import{S as Ru,i as Yu,s as Vu,e as s,k as p,w as d,t as n,M as Ju,c as l,d as a,m as u,a as o,x as m,h as r,b as h,G as t,g as c,y as f,q as v,o as y,B as _,v as zu}from"../chunks/vendor-hf-doc-builder.js";import{T as Ku}from"../chunks/Tip-hf-doc-builder.js";import{I as S}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as w}from"../chunks/CodeBlock-hf-doc-builder.js";function Qu(Ls){let g,le,k,P,Y;return{c(){g=s("p"),le=n("Note that features always describe the type of a single input element. In general we will add lists of elements so you can always think of a list around the types in "),k=s("code"),P=n("features"),Y=n(". Evaluate accepts various input formats (Python lists, NumPy arrays, PyTorch tensors, etc.) and converts them to an appropriate format for storage and computation.")},l(A){g=l(A,"P",{});var N=o(g);le=r(N,"Note that features always describe the type of a single input element. In general we will add lists of elements so you can always think of a list around the types in "),k=l(N,"CODE",{});var oe=o(k);P=r(oe,"features"),oe.forEach(a),Y=r(N,". Evaluate accepts various input formats (Python lists, NumPy arrays, PyTorch tensors, etc.) and converts them to an appropriate format for storage and computation."),N.forEach(a)},m(A,N){c(A,g,N),t(g,le),t(g,k),t(k,P),t(g,Y)},d(A){A&&a(g)}}}function Xu(Ls){let g,le,k,P,Y,A,N,oe,Po,Hs,jt,Do,Us,V,ne,Xt,Se,Co,Zt,Oo,Ws,kt,Io,Fs,M,re,ea,So,No,Ne,Mo,Bo,Lo,ie,ta,Ho,Uo,Me,Wo,Fo,Go,ce,aa,Ro,Yo,Be,Vo,Jo,Gs,pe,zo,xt,Ko,Qo,Rs,J,ue,sa,Le,Xo,la,Zo,Ys,he,en,oa,tn,an,Vs,He,Js,Tt,sn,zs,Ue,Ks,z,de,na,We,ln,ra,on,Qs,qt,nn,Xs,Fe,Zs,me,rn,At,cn,pn,el,K,fe,ia,Ge,un,ca,hn,tl,ve,dn,Pt,mn,fn,al,Re,sl,Q,ye,pa,Ye,vn,ua,yn,ll,_e,_n,Dt,bn,gn,ol,be,ha,Ve,da,wn,En,ma,$n,jn,b,Je,fa,va,kn,xn,ya,Tn,qn,ze,_a,ba,An,Pn,ga,Dn,Cn,Ke,wa,Ea,On,In,Qe,Sn,$a,Nn,Mn,Bn,Xe,ja,ka,Ln,Hn,xa,Un,Wn,Ze,Ta,qa,Fn,Gn,Aa,Rn,Yn,et,Pa,Da,Vn,Jn,Ca,zn,Kn,tt,Oa,Ia,Qn,Xn,Sa,Zn,er,at,Na,Ma,tr,ar,Ba,sr,nl,ge,lr,La,or,nr,rl,st,il,we,rr,Ha,ir,cr,cl,lt,pl,Ct,pr,ul,ot,hl,Ee,dl,X,$e,Ua,nt,ur,Wa,hr,ml,Ot,dr,fl,je,Fa,mr,fr,Ga,vr,vl,x,yr,It,_r,br,St,gr,wr,Nt,Er,$r,Ra,jr,kr,yl,Z,ke,Ya,rt,xr,Mt,Tr,Va,qr,_l,D,Ar,Ja,Pr,Dr,za,Cr,Or,Ka,Ir,Sr,bl,it,gl,B,Nr,Qa,Mr,Br,Xa,Lr,Hr,wl,ee,xe,Za,ct,Ur,Te,Wr,es,Fr,Gr,ts,Rr,El,T,Yr,as,Vr,Jr,ss,zr,Kr,ls,Qr,Xr,os,Zr,ei,$l,pt,jl,C,ti,ns,ai,si,rs,li,oi,is,ni,ri,kl,ut,xl,Bt,ii,Tl,ht,ql,te,qe,cs,dt,ci,ps,pi,Al,q,ui,us,hi,di,hs,mi,fi,mt,vi,yi,ds,_i,bi,Pl,Lt,gi,Dl,L,wi,ms,Ei,$i,fs,ji,ki,Cl,Ht,xi,Ol,ae,Ae,vs,ft,Ti,ys,qi,Il,Pe,Ai,Ut,Pi,Di,Sl,vt,Nl,Wt,Ci,Ml,yt,Bl,Ft,Oi,Ll,De,Ii,Gt,Si,Ni,Hl,_t,Ul,se,Ce,_s,bt,Mi,bs,Bi,Wl,E,Li,Rt,Hi,Ui,gs,Wi,Fi,ws,Gi,Ri,Es,Yi,Vi,$s,Ji,zi,Fl,H,Ki,js,Qi,Xi,ks,Zi,ec,Gl,gt,Rl,U,tc,xs,ac,sc,Ts,lc,oc,Yl,wt,Vl,W,nc,qs,rc,ic,As,cc,pc,Jl,Et,zl,$,uc,Ps,hc,dc,Ds,mc,fc,Cs,vc,yc,Os,_c,bc,Is,gc,wc,Kl;return A=new S({}),Se=new S({}),Le=new S({}),He=new w({props:{code:`import evaluate
accuracy = evaluate.load("accuracy")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> evaluate
<span class="hljs-meta">&gt;&gt;&gt; </span>accuracy = evaluate.load(<span class="hljs-string">&quot;accuracy&quot;</span>)`}}),Ue=new w({props:{code:'word_length = evaluate.load("word_length", module_type="measurement")',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>word_length = evaluate.load(<span class="hljs-string">&quot;word_length&quot;</span>, module_type=<span class="hljs-string">&quot;measurement&quot;</span>)'}}),We=new S({}),Fe=new w({props:{code:'element_count = evaluate.load("lvwerra/element_count", module_type="measurement")',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>element_count = evaluate.load(<span class="hljs-string">&quot;lvwerra/element_count&quot;</span>, module_type=<span class="hljs-string">&quot;measurement&quot;</span>)'}}),Ge=new S({}),Re=new w({props:{code:`evaluate.list_evaluation_modules(
  module_type="comparison",
  include_community=False, 
  with_details=True)

>>> [{'name': 'mcnemar', 'type': 'comparison', 'community': False, 'likes': 1},
...  {'name': 'exact_match', 'type': 'comparison', 'community': False, 'likes': 0}]`,highlighted:`evaluate.list_evaluation_modules(
  module_type=<span class="hljs-string">&quot;comparison&quot;</span>,
  include_community=<span class="hljs-literal">False</span>, 
  with_details=<span class="hljs-literal">True</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>[{<span class="hljs-string">&#x27;name&#x27;</span>: <span class="hljs-string">&#x27;mcnemar&#x27;</span>, <span class="hljs-string">&#x27;type&#x27;</span>: <span class="hljs-string">&#x27;comparison&#x27;</span>, <span class="hljs-string">&#x27;community&#x27;</span>: <span class="hljs-literal">False</span>, <span class="hljs-string">&#x27;likes&#x27;</span>: <span class="hljs-number">1</span>},
<span class="hljs-meta">... </span> {<span class="hljs-string">&#x27;name&#x27;</span>: <span class="hljs-string">&#x27;exact_match&#x27;</span>, <span class="hljs-string">&#x27;type&#x27;</span>: <span class="hljs-string">&#x27;comparison&#x27;</span>, <span class="hljs-string">&#x27;community&#x27;</span>: <span class="hljs-literal">False</span>, <span class="hljs-string">&#x27;likes&#x27;</span>: <span class="hljs-number">0</span>}]`}}),Ye=new S({}),st=new w({props:{code:`accuracy = evaluate.load("accuracy")
accuracy.description`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>accuracy = evaluate.load(<span class="hljs-string">&quot;accuracy&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>accuracy.description
Accuracy <span class="hljs-keyword">is</span> the proportion of correct predictions among the total number of cases processed. It can be computed <span class="hljs-keyword">with</span>:
Accuracy = (TP + TN) / (TP + TN + FP + FN)
 Where:
TP: <span class="hljs-literal">True</span> positive
TN: <span class="hljs-literal">True</span> negative
FP: <span class="hljs-literal">False</span> positive
FN: <span class="hljs-literal">False</span> negative`}}),lt=new w({props:{code:"accuracy.citation",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>accuracy.citation
<span class="hljs-meta">@article{scikit-learn,</span>
  title={Scikit-learn: Machine Learning <span class="hljs-keyword">in</span> {P}ython},
  author={Pedregosa, F. <span class="hljs-keyword">and</span> Varoquaux, G. <span class="hljs-keyword">and</span> Gramfort, A. <span class="hljs-keyword">and</span> Michel, V.
         <span class="hljs-keyword">and</span> Thirion, B. <span class="hljs-keyword">and</span> Grisel, O. <span class="hljs-keyword">and</span> Blondel, M. <span class="hljs-keyword">and</span> Prettenhofer, P.
         <span class="hljs-keyword">and</span> Weiss, R. <span class="hljs-keyword">and</span> Dubourg, V. <span class="hljs-keyword">and</span> Vanderplas, J. <span class="hljs-keyword">and</span> Passos, A. <span class="hljs-keyword">and</span>
         Cournapeau, D. <span class="hljs-keyword">and</span> Brucher, M. <span class="hljs-keyword">and</span> Perrot, M. <span class="hljs-keyword">and</span> Duchesnay, E.},
  journal={Journal of Machine Learning Research},
  volume={<span class="hljs-number">12</span>},
  pages={<span class="hljs-number">2825</span>--<span class="hljs-number">2830</span>},
  year={<span class="hljs-number">2011</span>}
}`}}),ot=new w({props:{code:"accuracy.features",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>accuracy.features
{
    <span class="hljs-string">&#x27;predictions&#x27;</span>: Value(dtype=<span class="hljs-string">&#x27;int32&#x27;</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>),
    <span class="hljs-string">&#x27;references&#x27;</span>: Value(dtype=<span class="hljs-string">&#x27;int32&#x27;</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>)
}`}}),Ee=new Ku({props:{$$slots:{default:[Qu]},$$scope:{ctx:Ls}}}),nt=new S({}),rt=new S({}),it=new w({props:{code:"accuracy.compute(references=[0,1,0,1], predictions=[1,0,0,1])",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>accuracy.compute(references=[<span class="hljs-number">0</span>,<span class="hljs-number">1</span>,<span class="hljs-number">0</span>,<span class="hljs-number">1</span>], predictions=[<span class="hljs-number">1</span>,<span class="hljs-number">0</span>,<span class="hljs-number">0</span>,<span class="hljs-number">1</span>])
{<span class="hljs-string">&#x27;accuracy&#x27;</span>: <span class="hljs-number">0.5</span>}`}}),ct=new S({}),pt=new w({props:{code:`for ref, pred in zip([0,1,0,1], [1,0,0,1]):
    accuracy.add(references=ref, predictions=pred)
accuracy.compute()`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">for</span> ref, pred <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>([<span class="hljs-number">0</span>,<span class="hljs-number">1</span>,<span class="hljs-number">0</span>,<span class="hljs-number">1</span>], [<span class="hljs-number">1</span>,<span class="hljs-number">0</span>,<span class="hljs-number">0</span>,<span class="hljs-number">1</span>]):
<span class="hljs-meta">&gt;&gt;&gt; </span>    accuracy.add(references=ref, predictions=pred)
<span class="hljs-meta">&gt;&gt;&gt; </span>accuracy.compute()
{<span class="hljs-string">&#x27;accuracy&#x27;</span>: <span class="hljs-number">0.5</span>}`}}),ut=new w({props:{code:`for refs, preds in zip([[0,1],[0,1]], [[1,0],[0,1]]):
    accuracy.add_batch(references=refs, predictions=preds)
accuracy.compute()`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">for</span> refs, preds <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>([[<span class="hljs-number">0</span>,<span class="hljs-number">1</span>],[<span class="hljs-number">0</span>,<span class="hljs-number">1</span>]], [[<span class="hljs-number">1</span>,<span class="hljs-number">0</span>],[<span class="hljs-number">0</span>,<span class="hljs-number">1</span>]]):
<span class="hljs-meta">&gt;&gt;&gt; </span>    accuracy.add_batch(references=refs, predictions=preds)
<span class="hljs-meta">&gt;&gt;&gt; </span>accuracy.compute()
{<span class="hljs-string">&#x27;accuracy&#x27;</span>: <span class="hljs-number">0.5</span>}`}}),ht=new w({props:{code:`for model_inputs, gold_standards in evaluation_dataset:
    predictions = model(model_inputs)
    metric.add_batch(references=gold_standards, predictions=predictions)
metric.compute()`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">for</span> model_inputs, gold_standards <span class="hljs-keyword">in</span> evaluation_dataset:
<span class="hljs-meta">&gt;&gt;&gt; </span>    predictions = model(model_inputs)
<span class="hljs-meta">&gt;&gt;&gt; </span>    metric.add_batch(references=gold_standards, predictions=predictions)
<span class="hljs-meta">&gt;&gt;&gt; </span>metric.compute()`}}),dt=new S({}),ft=new S({}),vt=new w({props:{code:`result = accuracy.compute(references=[0,1,0,1], predictions=[1,0,0,1])

hyperparams = {"model": "bert-base-uncased"}
evaluate.save("./results/"experiment="run 42", **result, **hyperparams)
>>> PosixPath('results/result-2022_05_30-22_09_11.json')`,highlighted:`result = accuracy.compute(references=[<span class="hljs-number">0</span>,<span class="hljs-number">1</span>,<span class="hljs-number">0</span>,<span class="hljs-number">1</span>], predictions=[<span class="hljs-number">1</span>,<span class="hljs-number">0</span>,<span class="hljs-number">0</span>,<span class="hljs-number">1</span>])

hyperparams = {<span class="hljs-string">&quot;model&quot;</span>: <span class="hljs-string">&quot;bert-base-uncased&quot;</span>}
evaluate.save(<span class="hljs-string">&quot;./results/&quot;</span>experiment=<span class="hljs-string">&quot;run 42&quot;</span>, **result, **hyperparams)
<span class="hljs-meta">&gt;&gt;&gt; </span>PosixPath(<span class="hljs-string">&#x27;results/result-2022_05_30-22_09_11.json&#x27;</span>)`}}),yt=new w({props:{code:`{
    "experiment": "run 42",
    "accuracy": 0.5,
    "model": "bert-base-uncased",
    "_timestamp": "2022-05-30T22:09:11.959469",
    "_git_commit_hash": "123456789abcdefghijkl",
    "_evaluate_version": "0.1.0",
    "_python_version": "3.9.12 (main, Mar 26 2022, 15:51:15) \\n[Clang 13.1.6 (clang-1316.0.21.2)]",
    "_interpreter_path": "/Users/leandro/git/evaluate/env/bin/python"
}`,highlighted:`<span class="hljs-punctuation">{</span>
    <span class="hljs-attr">&quot;experiment&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;run 42&quot;</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;accuracy&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">0.5</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;model&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;bert-base-uncased&quot;</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;_timestamp&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;2022-05-30T22:09:11.959469&quot;</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;_git_commit_hash&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;123456789abcdefghijkl&quot;</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;_evaluate_version&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;0.1.0&quot;</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;_python_version&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;3.9.12 (main, Mar 26 2022, 15:51:15) \\n[Clang 13.1.6 (clang-1316.0.21.2)]&quot;</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;_interpreter_path&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;/Users/leandro/git/evaluate/env/bin/python&quot;</span>
<span class="hljs-punctuation">}</span>`}}),_t=new w({props:{code:`evaluate.push_to_hub(
  model_id="huggingface/gpt2-wikitext2",  # model repository on hub
  metric_value=0.5,                       # metric value
  metric_type="bleu",                     # metric name, e.g. accuracy.name
  metric_name="BLEU",                     # pretty name which is displayed
  dataset_type="wikitext",                # dataset name on the hub
  dataset_name="WikiText",                # pretty name
  dataset_split="test",                   # dataset split used
  task_type="text-generation",            # task id, see https://github.com/huggingface/datasets/blob/master/src/datasets/utils/resources/tasks.json
  task_name="Text Generation"             # pretty name for task
)`,highlighted:`evaluate.push_to_hub(
  model_id=<span class="hljs-string">&quot;huggingface/gpt2-wikitext2&quot;</span>,  <span class="hljs-comment"># model repository on hub</span>
  metric_value=<span class="hljs-number">0.5</span>,                       <span class="hljs-comment"># metric value</span>
  metric_type=<span class="hljs-string">&quot;bleu&quot;</span>,                     <span class="hljs-comment"># metric name, e.g. accuracy.name</span>
  metric_name=<span class="hljs-string">&quot;BLEU&quot;</span>,                     <span class="hljs-comment"># pretty name which is displayed</span>
  dataset_type=<span class="hljs-string">&quot;wikitext&quot;</span>,                <span class="hljs-comment"># dataset name on the hub</span>
  dataset_name=<span class="hljs-string">&quot;WikiText&quot;</span>,                <span class="hljs-comment"># pretty name</span>
  dataset_split=<span class="hljs-string">&quot;test&quot;</span>,                   <span class="hljs-comment"># dataset split used</span>
  task_type=<span class="hljs-string">&quot;text-generation&quot;</span>,            <span class="hljs-comment"># task id, see https://github.com/huggingface/datasets/blob/master/src/datasets/utils/resources/tasks.json</span>
  task_name=<span class="hljs-string">&quot;Text Generation&quot;</span>             <span class="hljs-comment"># pretty name for task</span>
)`}}),bt=new S({}),gt=new w({props:{code:`from transformers import pipeline
from datasets import load_dataset
from evaluate import evaluator
import evaluate

pipe = pipeline("text-classification", model="lvwerra/distilbert-imdb", device=0)
data = load_dataset("imdb", split="test").shuffle().select(range(1000))
metric = evaluate.load("accuracy")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline
<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-keyword">from</span> evaluate <span class="hljs-keyword">import</span> evaluator
<span class="hljs-keyword">import</span> evaluate

pipe = pipeline(<span class="hljs-string">&quot;text-classification&quot;</span>, model=<span class="hljs-string">&quot;lvwerra/distilbert-imdb&quot;</span>, device=<span class="hljs-number">0</span>)
data = load_dataset(<span class="hljs-string">&quot;imdb&quot;</span>, split=<span class="hljs-string">&quot;test&quot;</span>).shuffle().select(<span class="hljs-built_in">range</span>(<span class="hljs-number">1000</span>))
metric = evaluate.load(<span class="hljs-string">&quot;accuracy&quot;</span>)`}}),wt=new w({props:{code:`eval = evaluator("text-classification")

results = eval.compute(model_or_pipeline=pipe, data=data, metric=metric,
                       label_mapping={"NEGATIVE": 0, "POSITIVE": 1},)

print(results)
>>> {'accuracy': 0.934}`,highlighted:`<span class="hljs-built_in">eval</span> = evaluator(<span class="hljs-string">&quot;text-classification&quot;</span>)

results = <span class="hljs-built_in">eval</span>.compute(model_or_pipeline=pipe, data=data, metric=metric,
                       label_mapping={<span class="hljs-string">&quot;NEGATIVE&quot;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&quot;POSITIVE&quot;</span>: <span class="hljs-number">1</span>},)

<span class="hljs-built_in">print</span>(results)
<span class="hljs-meta">&gt;&gt;&gt; </span>{<span class="hljs-string">&#x27;accuracy&#x27;</span>: <span class="hljs-number">0.934</span>}`}}),Et=new w({props:{code:`results = eval.compute(model_or_pipeline=pipe, data=data, metric=metric,
                       label_mapping={"NEGATIVE": 0, "POSITIVE": 1},
                       strategy="bootstrap", n_resamples=200)

print(results)
>>> {'accuracy': 
...     {
...       'confidence_interval': (0.906, 0.9406749892841922),
...       'standard_error': 0.00865213251082787,
...       'score': 0.923
...     }
... }`,highlighted:`results = <span class="hljs-built_in">eval</span>.compute(model_or_pipeline=pipe, data=data, metric=metric,
                       label_mapping={<span class="hljs-string">&quot;NEGATIVE&quot;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&quot;POSITIVE&quot;</span>: <span class="hljs-number">1</span>},
                       strategy=<span class="hljs-string">&quot;bootstrap&quot;</span>, n_resamples=<span class="hljs-number">200</span>)

<span class="hljs-built_in">print</span>(results)
<span class="hljs-meta">&gt;&gt;&gt; </span>{<span class="hljs-string">&#x27;accuracy&#x27;</span>: 
<span class="hljs-meta">... </span>    {
<span class="hljs-meta">... </span>      <span class="hljs-string">&#x27;confidence_interval&#x27;</span>: (<span class="hljs-number">0.906</span>, <span class="hljs-number">0.9406749892841922</span>),
<span class="hljs-meta">... </span>      <span class="hljs-string">&#x27;standard_error&#x27;</span>: <span class="hljs-number">0.00865213251082787</span>,
<span class="hljs-meta">... </span>      <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.923</span>
<span class="hljs-meta">... </span>    }
<span class="hljs-meta">... </span>}`}}),{c(){g=s("meta"),le=p(),k=s("h1"),P=s("a"),Y=s("span"),d(A.$$.fragment),N=p(),oe=s("span"),Po=n("A quick tour"),Hs=p(),jt=s("p"),Do=n("\u{1F917} Evaluate provides access to a wide range of evaluation tools. It covers a range of modalities such as text, computer vision, audio, etc. as well as tools to evaluate models or datasets. These tools are split into three categories."),Us=p(),V=s("h2"),ne=s("a"),Xt=s("span"),d(Se.$$.fragment),Co=p(),Zt=s("span"),Oo=n("Types of evaluations"),Ws=p(),kt=s("p"),Io=n("There are different aspects of a typical machine learning pipeline that can be evaluated and for each aspect \u{1F917} Evaluate provides a tool:"),Fs=p(),M=s("ul"),re=s("li"),ea=s("strong"),So=n("Metric"),No=n(": A metric is used to evaluate a model\u2019s performance and usually involves the model\u2019s predictions as well as some ground truth labels. You can find all integrated metrics at "),Ne=s("a"),Mo=n("evaluate-metric"),Bo=n("."),Lo=p(),ie=s("li"),ta=s("strong"),Ho=n("Comparison"),Uo=n(": A comparison is used to compare two models. This can for example be done by comparing their predictions to ground truth labels and computing their agreement. You can find all integrated comparisons at "),Me=s("a"),Wo=n("evaluate-comparison"),Fo=n("."),Go=p(),ce=s("li"),aa=s("strong"),Ro=n("Measurement"),Yo=n(": The dataset is as important as the model trained on it. With measurements one can investigate a dataset\u2019s properties. You can find all integrated measurements at "),Be=s("a"),Vo=n("evaluate-measurement"),Jo=n("."),Gs=p(),pe=s("p"),zo=n("Each metric, comparison, and measurement is a separate Python module, but for using any of them, there is a single entry point: "),xt=s("a"),Ko=n("evaluate.load()"),Qo=n("!"),Rs=p(),J=s("h2"),ue=s("a"),sa=s("span"),d(Le.$$.fragment),Xo=p(),la=s("span"),Zo=n("Load"),Ys=p(),he=s("p"),en=n("Any metric, comparison, or measurement is loaded with the "),oa=s("code"),tn=n("evaluate.load"),an=n(" function:"),Vs=p(),d(He.$$.fragment),Js=p(),Tt=s("p"),sn=n("If you want to make sure you are loading the right type of evaluation (especially if there are name clashes) you can explicitely pass the type:"),zs=p(),d(Ue.$$.fragment),Ks=p(),z=s("h3"),de=s("a"),na=s("span"),d(We.$$.fragment),ln=p(),ra=s("span"),on=n("Community modules"),Qs=p(),qt=s("p"),nn=n("Besides the modules implemented in \u{1F917} Evaluate you can also load any community module by specifying the repository ID of the metric implementation:"),Xs=p(),d(Fe.$$.fragment),Zs=p(),me=s("p"),rn=n("See the "),At=s("a"),cn=n("Creating and Sharing Guide"),pn=n(" for information about uploading custom metrics."),el=p(),K=s("h3"),fe=s("a"),ia=s("span"),d(Ge.$$.fragment),un=p(),ca=s("span"),hn=n("List available modules"),tl=p(),ve=s("p"),dn=n("With "),Pt=s("a"),mn=n("list_evaluation_modules()"),fn=n(" you can check what modules are available on the hub. You can also filter for a specific modules and skip community metrics if you want. You can also see additional information such as likes:"),al=p(),d(Re.$$.fragment),sl=p(),Q=s("h2"),ye=s("a"),pa=s("span"),d(Ye.$$.fragment),vn=p(),ua=s("span"),yn=n("Module attributes"),ll=p(),_e=s("p"),_n=n("All evalution modules come with a range of useful attributes that help to use a module stored in a "),Dt=s("a"),bn=n("EvaluationModuleInfo"),gn=n(" object."),ol=p(),be=s("table"),ha=s("thead"),Ve=s("tr"),da=s("th"),wn=n("Attribute"),En=p(),ma=s("th"),$n=n("Description"),jn=p(),b=s("tbody"),Je=s("tr"),fa=s("td"),va=s("code"),kn=n("description"),xn=p(),ya=s("td"),Tn=n("A short description of the evaluation module."),qn=p(),ze=s("tr"),_a=s("td"),ba=s("code"),An=n("citation"),Pn=p(),ga=s("td"),Dn=n("A BibTex string for citation when available."),Cn=p(),Ke=s("tr"),wa=s("td"),Ea=s("code"),On=n("features"),In=p(),Qe=s("td"),Sn=n("A "),$a=s("code"),Nn=n("Features"),Mn=n(" object defining the input format."),Bn=p(),Xe=s("tr"),ja=s("td"),ka=s("code"),Ln=n("inputs_description"),Hn=p(),xa=s("td"),Un=n("This is equivalent to the modules docstring."),Wn=p(),Ze=s("tr"),Ta=s("td"),qa=s("code"),Fn=n("homepage"),Gn=p(),Aa=s("td"),Rn=n("The homepage of the module."),Yn=p(),et=s("tr"),Pa=s("td"),Da=s("code"),Vn=n("license"),Jn=p(),Ca=s("td"),zn=n("The license of the module."),Kn=p(),tt=s("tr"),Oa=s("td"),Ia=s("code"),Qn=n("codebase_urls"),Xn=p(),Sa=s("td"),Zn=n("Link to the code behind the module."),er=p(),at=s("tr"),Na=s("td"),Ma=s("code"),tr=n("reference_urls"),ar=p(),Ba=s("td"),sr=n("Additional reference URLs."),nl=p(),ge=s("p"),lr=n("Let\u2019s have a look at a few examples. First, let\u2019s look at the "),La=s("code"),or=n("description"),nr=n(" attribute of the accuracy metric:"),rl=p(),d(st.$$.fragment),il=p(),we=s("p"),rr=n("You can see that it describes how the metric works in theory. If you use this metric for your work, especially if it is an academic publication you want to reference it properly. For that you can look at the "),Ha=s("code"),ir=n("citation"),cr=n(" attribute:"),cl=p(),d(lt.$$.fragment),pl=p(),Ct=s("p"),pr=n("Before we can apply a metric or other evaluation module to a use-case, we need to know what the input format of the metric is:"),ul=p(),d(ot.$$.fragment),hl=p(),d(Ee.$$.fragment),dl=p(),X=s("h2"),$e=s("a"),Ua=s("span"),d(nt.$$.fragment),ur=p(),Wa=s("span"),hr=n("Compute"),ml=p(),Ot=s("p"),dr=n("Now that we know how the evaluation module works and what should go in there we want to actually use it! When it comes to computing the actual score there are two main ways to do it:"),fl=p(),je=s("ol"),Fa=s("li"),mr=n("All-in-one"),fr=p(),Ga=s("li"),vr=n("Incremental"),vl=p(),x=s("p"),yr=n("In the incremental approach the necessary inputs are added to the module with "),It=s("a"),_r=n("EvaluationModule.add()"),br=n(" or "),St=s("a"),gr=n("EvaluationModule.add_batch()"),wr=n(" and the score is calculated at the end with "),Nt=s("a"),Er=n("EvaluationModule.compute()"),$r=n(". Alternatively, one can pass all the inputs at once to "),Ra=s("code"),jr=n("compute()"),kr=n(". Let\u2019s have a look at the two approaches."),yl=p(),Z=s("h3"),ke=s("a"),Ya=s("span"),d(rt.$$.fragment),xr=p(),Mt=s("span"),Tr=n("Using "),Va=s("code"),qr=n("compute()"),_l=p(),D=s("p"),Ar=n("The simplest way to calculate the score of an evaluation module is by calling "),Ja=s("code"),Pr=n("compute()"),Dr=n(" directly with the necessary inputs. Simply pass the inputs as seen in "),za=s("code"),Cr=n("features"),Or=n(" to the "),Ka=s("code"),Ir=n("compute()"),Sr=n(" method."),bl=p(),d(it.$$.fragment),gl=p(),B=s("p"),Nr=n("Evaluation modules return the results in a dictionary. However, in some instances you build up the predictions iteratively or in a distributed fashion in which case "),Qa=s("code"),Mr=n("add()"),Br=n(" or "),Xa=s("code"),Lr=n("add_batch()"),Hr=n(" are useful."),wl=p(),ee=s("h3"),xe=s("a"),Za=s("span"),d(ct.$$.fragment),Ur=p(),Te=s("span"),Wr=n("Using "),es=s("code"),Fr=n("add()"),Gr=n(" and "),ts=s("code"),Rr=n("add_batch()"),El=p(),T=s("p"),Yr=n("In many evaluation pipelines you build the predictions iteratively such as in a for-loop. In that case you could store the predictions in a list and at the end pass them to "),as=s("code"),Vr=n("compute()"),Jr=n(". With "),ss=s("code"),zr=n("add()"),Kr=n(" and "),ls=s("code"),Qr=n("add_batch()"),Xr=n(" you can circumvent the step of storing the predictions separately. If you are only creating single predictions at a time you can use "),os=s("code"),Zr=n("add()"),ei=n(":"),$l=p(),d(pt.$$.fragment),jl=p(),C=s("p"),ti=n("Once you have gathered all predictions you can call "),ns=s("code"),ai=n("compute()"),si=n(" to compute the score based on all stored values. When getting predictions and references in batches you can use "),rs=s("code"),li=n("add_batch()"),oi=n(" which adds a list elements for later processing. The rest works as with "),is=s("code"),ni=n("add()"),ri=n(":"),kl=p(),d(ut.$$.fragment),xl=p(),Bt=s("p"),ii=n("This is especially useful when you need to get the predictions from your model in batches:"),Tl=p(),d(ht.$$.fragment),ql=p(),te=s("h3"),qe=s("a"),cs=s("span"),d(dt.$$.fragment),ci=p(),ps=s("span"),pi=n("Distributed evaluation"),Al=p(),q=s("p"),ui=n("Computing metrics in a distributed environment can be tricky. Metric evaluation is executed in separate Python processes, or nodes, on different subsets of a dataset. Typically, when a metric score is additive ("),us=s("code"),hi=n("f(AuB) = f(A) + f(B)"),di=n("), you can use distributed reduce operations to gather the scores for each subset of the dataset. But when a metric is non-additive ("),hs=s("code"),mi=n("f(AuB) \u2260 f(A) + f(B)"),fi=n("), it\u2019s not that simple. For example, you can\u2019t take the sum of the "),mt=s("a"),vi=n("F1"),yi=n(" scores of each data subset as your "),ds=s("strong"),_i=n("final metric"),bi=n("."),Pl=p(),Lt=s("p"),gi=n("A common way to overcome this issue is to fallback on single process evaluation. The metrics are evaluated on a single GPU, which becomes inefficient."),Dl=p(),L=s("p"),wi=n("\u{1F917} Evaluate solves this issue by only computing the final metric on the first node. The predictions and references are computed and provided to the metric separately for each node. These are temporarily stored in an Apache Arrow table, avoiding cluttering the GPU or CPU memory. When you are ready to "),ms=s("code"),Ei=n("compute()"),$i=n(" the final metric, the first node is able to access the predictions and references stored on all the other nodes. Once it has gathered all the predictions and references, "),fs=s("code"),ji=n("compute()"),ki=n(" will perform the final metric evaluation."),Cl=p(),Ht=s("p"),xi=n("This solution allows \u{1F917} Evaluate to perform distributed predictions, which is important for evaluation speed in distributed settings. At the same time, you can also use complex non-additive metrics without wasting valuable GPU or CPU memory."),Ol=p(),ae=s("h2"),Ae=s("a"),vs=s("span"),d(ft.$$.fragment),Ti=p(),ys=s("span"),qi=n("Save and push to the Hub"),Il=p(),Pe=s("p"),Ai=n("Saving and sharing evaluation results is an important step. We provide the "),Ut=s("a"),Pi=n("evaluate.save()"),Di=n(" function to easily save metrics results. You can either pass a specific filename or a directory. In the latter case, the results are saved in a file with an automatically created file name. Besides the directory or file name, the function takes any key-value pairs as inputs and stores them in a JSON file."),Sl=p(),d(vt.$$.fragment),Nl=p(),Wt=s("p"),Ci=n("The content of the JSON file look like the following:"),Ml=p(),d(yt.$$.fragment),Bl=p(),Ft=s("p"),Oi=n("In addition to the specified fields, it also contains useful system information for reproducing the results."),Ll=p(),De=s("p"),Ii=n("Besides storing the results locally, you should report them on the model\u2019s repository on the Hub. With the "),Gt=s("a"),Si=n("evaluate.push_to_hub()"),Ni=n(" function, you can easily report evaluation results to the model\u2019s repository:"),Hl=p(),d(_t.$$.fragment),Ul=p(),se=s("h2"),Ce=s("a"),_s=s("span"),d(bt.$$.fragment),Mi=p(),bs=s("span"),Bi=n("Evaluator"),Wl=p(),E=s("p"),Li=n("The "),Rt=s("a"),Hi=n("evaluate.evaluator()"),Ui=n(" provides automated evaluation and only requires a model, dataset, metric in contrast to the metrics in "),gs=s("code"),Wi=n("EvaluationModule"),Fi=n("s that require the model\u2019s predictions. As such it is easier to evaluate a model on a dataset with a given metric as the inference is handled internally. To make that possible it uses the "),ws=s("code"),Gi=n("Pipeline"),Ri=n(" abstraction from "),Es=s("code"),Yi=n("transformers"),Vi=n(". However, you can use your own framework as long as it follows the "),$s=s("code"),Ji=n("Pipeline"),zi=n(" interface."),Fl=p(),H=s("p"),Ki=n("To make an evaluation with the "),js=s("code"),Qi=n("evaluator"),Xi=n(" let\u2019s load a "),ks=s("code"),Zi=n("transformers"),ec=n(" pipeline (but you can pass your own custom inference class for any framework as long as it follows the pipeline call API) with an model trained on IMDb, the IMDb test split and the accuracy metric."),Gl=p(),d(gt.$$.fragment),Rl=p(),U=s("p"),tc=n("Then you can create an evaluator for text classification and pass the three objects to the "),xs=s("code"),ac=n("compute()"),sc=n(" method. With the label mapping "),Ts=s("code"),lc=n("evaluate"),oc=n(" provides a method to align the pipeline outputs with the label column in the dataset:"),Yl=p(),d(wt.$$.fragment),Vl=p(),W=s("p"),nc=n("Calculating the value of the metric alone is often not enough to know if a model performs significantly better than another one. With "),qs=s("em"),rc=n("bootstrapping"),ic=p(),As=s("code"),cc=n("evaluate"),pc=n(" computes confidence intervals and the standard error which helps estimate how stable a score is:"),Jl=p(),d(Et.$$.fragment),zl=p(),$=s("p"),uc=n("The evaluator expects a "),Ps=s("code"),hc=n('"text"'),dc=n(" and "),Ds=s("code"),mc=n('"label"'),fc=n(" column for the data input. If your dataset differs you can provide the columns with the keywords "),Cs=s("code"),vc=n('input_column="text"'),yc=n(" and "),Os=s("code"),_c=n('label_column="label"'),bc=n(". Currently only "),Is=s("code"),gc=n('"text-classification"'),wc=n(" is supported with more tasks being added in the future."),this.h()},l(e){const i=Ju('[data-svelte="svelte-1phssyn"]',document.head);g=l(i,"META",{name:!0,content:!0}),i.forEach(a),le=u(e),k=l(e,"H1",{class:!0});var $t=o(k);P=l($t,"A",{id:!0,class:!0,href:!0});var $c=o(P);Y=l($c,"SPAN",{});var jc=o(Y);m(A.$$.fragment,jc),jc.forEach(a),$c.forEach(a),N=u($t),oe=l($t,"SPAN",{});var kc=o(oe);Po=r(kc,"A quick tour"),kc.forEach(a),$t.forEach(a),Hs=u(e),jt=l(e,"P",{});var xc=o(jt);Do=r(xc,"\u{1F917} Evaluate provides access to a wide range of evaluation tools. It covers a range of modalities such as text, computer vision, audio, etc. as well as tools to evaluate models or datasets. These tools are split into three categories."),xc.forEach(a),Us=u(e),V=l(e,"H2",{class:!0});var Ql=o(V);ne=l(Ql,"A",{id:!0,class:!0,href:!0});var Tc=o(ne);Xt=l(Tc,"SPAN",{});var qc=o(Xt);m(Se.$$.fragment,qc),qc.forEach(a),Tc.forEach(a),Co=u(Ql),Zt=l(Ql,"SPAN",{});var Ac=o(Zt);Oo=r(Ac,"Types of evaluations"),Ac.forEach(a),Ql.forEach(a),Ws=u(e),kt=l(e,"P",{});var Pc=o(kt);Io=r(Pc,"There are different aspects of a typical machine learning pipeline that can be evaluated and for each aspect \u{1F917} Evaluate provides a tool:"),Pc.forEach(a),Fs=u(e),M=l(e,"UL",{});var Yt=o(M);re=l(Yt,"LI",{});var Ss=o(re);ea=l(Ss,"STRONG",{});var Dc=o(ea);So=r(Dc,"Metric"),Dc.forEach(a),No=r(Ss,": A metric is used to evaluate a model\u2019s performance and usually involves the model\u2019s predictions as well as some ground truth labels. You can find all integrated metrics at "),Ne=l(Ss,"A",{href:!0,rel:!0});var Cc=o(Ne);Mo=r(Cc,"evaluate-metric"),Cc.forEach(a),Bo=r(Ss,"."),Ss.forEach(a),Lo=u(Yt),ie=l(Yt,"LI",{});var Ns=o(ie);ta=l(Ns,"STRONG",{});var Oc=o(ta);Ho=r(Oc,"Comparison"),Oc.forEach(a),Uo=r(Ns,": A comparison is used to compare two models. This can for example be done by comparing their predictions to ground truth labels and computing their agreement. You can find all integrated comparisons at "),Me=l(Ns,"A",{href:!0,rel:!0});var Ic=o(Me);Wo=r(Ic,"evaluate-comparison"),Ic.forEach(a),Fo=r(Ns,"."),Ns.forEach(a),Go=u(Yt),ce=l(Yt,"LI",{});var Ms=o(ce);aa=l(Ms,"STRONG",{});var Sc=o(aa);Ro=r(Sc,"Measurement"),Sc.forEach(a),Yo=r(Ms,": The dataset is as important as the model trained on it. With measurements one can investigate a dataset\u2019s properties. You can find all integrated measurements at "),Be=l(Ms,"A",{href:!0,rel:!0});var Nc=o(Be);Vo=r(Nc,"evaluate-measurement"),Nc.forEach(a),Jo=r(Ms,"."),Ms.forEach(a),Yt.forEach(a),Gs=u(e),pe=l(e,"P",{});var Xl=o(pe);zo=r(Xl,"Each metric, comparison, and measurement is a separate Python module, but for using any of them, there is a single entry point: "),xt=l(Xl,"A",{href:!0});var Mc=o(xt);Ko=r(Mc,"evaluate.load()"),Mc.forEach(a),Qo=r(Xl,"!"),Xl.forEach(a),Rs=u(e),J=l(e,"H2",{class:!0});var Zl=o(J);ue=l(Zl,"A",{id:!0,class:!0,href:!0});var Bc=o(ue);sa=l(Bc,"SPAN",{});var Lc=o(sa);m(Le.$$.fragment,Lc),Lc.forEach(a),Bc.forEach(a),Xo=u(Zl),la=l(Zl,"SPAN",{});var Hc=o(la);Zo=r(Hc,"Load"),Hc.forEach(a),Zl.forEach(a),Ys=u(e),he=l(e,"P",{});var eo=o(he);en=r(eo,"Any metric, comparison, or measurement is loaded with the "),oa=l(eo,"CODE",{});var Uc=o(oa);tn=r(Uc,"evaluate.load"),Uc.forEach(a),an=r(eo," function:"),eo.forEach(a),Vs=u(e),m(He.$$.fragment,e),Js=u(e),Tt=l(e,"P",{});var Wc=o(Tt);sn=r(Wc,"If you want to make sure you are loading the right type of evaluation (especially if there are name clashes) you can explicitely pass the type:"),Wc.forEach(a),zs=u(e),m(Ue.$$.fragment,e),Ks=u(e),z=l(e,"H3",{class:!0});var to=o(z);de=l(to,"A",{id:!0,class:!0,href:!0});var Fc=o(de);na=l(Fc,"SPAN",{});var Gc=o(na);m(We.$$.fragment,Gc),Gc.forEach(a),Fc.forEach(a),ln=u(to),ra=l(to,"SPAN",{});var Rc=o(ra);on=r(Rc,"Community modules"),Rc.forEach(a),to.forEach(a),Qs=u(e),qt=l(e,"P",{});var Yc=o(qt);nn=r(Yc,"Besides the modules implemented in \u{1F917} Evaluate you can also load any community module by specifying the repository ID of the metric implementation:"),Yc.forEach(a),Xs=u(e),m(Fe.$$.fragment,e),Zs=u(e),me=l(e,"P",{});var ao=o(me);rn=r(ao,"See the "),At=l(ao,"A",{href:!0});var Vc=o(At);cn=r(Vc,"Creating and Sharing Guide"),Vc.forEach(a),pn=r(ao," for information about uploading custom metrics."),ao.forEach(a),el=u(e),K=l(e,"H3",{class:!0});var so=o(K);fe=l(so,"A",{id:!0,class:!0,href:!0});var Jc=o(fe);ia=l(Jc,"SPAN",{});var zc=o(ia);m(Ge.$$.fragment,zc),zc.forEach(a),Jc.forEach(a),un=u(so),ca=l(so,"SPAN",{});var Kc=o(ca);hn=r(Kc,"List available modules"),Kc.forEach(a),so.forEach(a),tl=u(e),ve=l(e,"P",{});var lo=o(ve);dn=r(lo,"With "),Pt=l(lo,"A",{href:!0});var Qc=o(Pt);mn=r(Qc,"list_evaluation_modules()"),Qc.forEach(a),fn=r(lo," you can check what modules are available on the hub. You can also filter for a specific modules and skip community metrics if you want. You can also see additional information such as likes:"),lo.forEach(a),al=u(e),m(Re.$$.fragment,e),sl=u(e),Q=l(e,"H2",{class:!0});var oo=o(Q);ye=l(oo,"A",{id:!0,class:!0,href:!0});var Xc=o(ye);pa=l(Xc,"SPAN",{});var Zc=o(pa);m(Ye.$$.fragment,Zc),Zc.forEach(a),Xc.forEach(a),vn=u(oo),ua=l(oo,"SPAN",{});var ep=o(ua);yn=r(ep,"Module attributes"),ep.forEach(a),oo.forEach(a),ll=u(e),_e=l(e,"P",{});var no=o(_e);_n=r(no,"All evalution modules come with a range of useful attributes that help to use a module stored in a "),Dt=l(no,"A",{href:!0});var tp=o(Dt);bn=r(tp,"EvaluationModuleInfo"),tp.forEach(a),gn=r(no," object."),no.forEach(a),ol=u(e),be=l(e,"TABLE",{});var ro=o(be);ha=l(ro,"THEAD",{});var ap=o(ha);Ve=l(ap,"TR",{});var io=o(Ve);da=l(io,"TH",{});var sp=o(da);wn=r(sp,"Attribute"),sp.forEach(a),En=u(io),ma=l(io,"TH",{});var lp=o(ma);$n=r(lp,"Description"),lp.forEach(a),io.forEach(a),ap.forEach(a),jn=u(ro),b=l(ro,"TBODY",{});var j=o(b);Je=l(j,"TR",{});var co=o(Je);fa=l(co,"TD",{});var op=o(fa);va=l(op,"CODE",{});var np=o(va);kn=r(np,"description"),np.forEach(a),op.forEach(a),xn=u(co),ya=l(co,"TD",{});var rp=o(ya);Tn=r(rp,"A short description of the evaluation module."),rp.forEach(a),co.forEach(a),qn=u(j),ze=l(j,"TR",{});var po=o(ze);_a=l(po,"TD",{});var ip=o(_a);ba=l(ip,"CODE",{});var cp=o(ba);An=r(cp,"citation"),cp.forEach(a),ip.forEach(a),Pn=u(po),ga=l(po,"TD",{});var pp=o(ga);Dn=r(pp,"A BibTex string for citation when available."),pp.forEach(a),po.forEach(a),Cn=u(j),Ke=l(j,"TR",{});var uo=o(Ke);wa=l(uo,"TD",{});var up=o(wa);Ea=l(up,"CODE",{});var hp=o(Ea);On=r(hp,"features"),hp.forEach(a),up.forEach(a),In=u(uo),Qe=l(uo,"TD",{});var ho=o(Qe);Sn=r(ho,"A "),$a=l(ho,"CODE",{});var dp=o($a);Nn=r(dp,"Features"),dp.forEach(a),Mn=r(ho," object defining the input format."),ho.forEach(a),uo.forEach(a),Bn=u(j),Xe=l(j,"TR",{});var mo=o(Xe);ja=l(mo,"TD",{});var mp=o(ja);ka=l(mp,"CODE",{});var fp=o(ka);Ln=r(fp,"inputs_description"),fp.forEach(a),mp.forEach(a),Hn=u(mo),xa=l(mo,"TD",{});var vp=o(xa);Un=r(vp,"This is equivalent to the modules docstring."),vp.forEach(a),mo.forEach(a),Wn=u(j),Ze=l(j,"TR",{});var fo=o(Ze);Ta=l(fo,"TD",{});var yp=o(Ta);qa=l(yp,"CODE",{});var _p=o(qa);Fn=r(_p,"homepage"),_p.forEach(a),yp.forEach(a),Gn=u(fo),Aa=l(fo,"TD",{});var bp=o(Aa);Rn=r(bp,"The homepage of the module."),bp.forEach(a),fo.forEach(a),Yn=u(j),et=l(j,"TR",{});var vo=o(et);Pa=l(vo,"TD",{});var gp=o(Pa);Da=l(gp,"CODE",{});var wp=o(Da);Vn=r(wp,"license"),wp.forEach(a),gp.forEach(a),Jn=u(vo),Ca=l(vo,"TD",{});var Ep=o(Ca);zn=r(Ep,"The license of the module."),Ep.forEach(a),vo.forEach(a),Kn=u(j),tt=l(j,"TR",{});var yo=o(tt);Oa=l(yo,"TD",{});var $p=o(Oa);Ia=l($p,"CODE",{});var jp=o(Ia);Qn=r(jp,"codebase_urls"),jp.forEach(a),$p.forEach(a),Xn=u(yo),Sa=l(yo,"TD",{});var kp=o(Sa);Zn=r(kp,"Link to the code behind the module."),kp.forEach(a),yo.forEach(a),er=u(j),at=l(j,"TR",{});var _o=o(at);Na=l(_o,"TD",{});var xp=o(Na);Ma=l(xp,"CODE",{});var Tp=o(Ma);tr=r(Tp,"reference_urls"),Tp.forEach(a),xp.forEach(a),ar=u(_o),Ba=l(_o,"TD",{});var qp=o(Ba);sr=r(qp,"Additional reference URLs."),qp.forEach(a),_o.forEach(a),j.forEach(a),ro.forEach(a),nl=u(e),ge=l(e,"P",{});var bo=o(ge);lr=r(bo,"Let\u2019s have a look at a few examples. First, let\u2019s look at the "),La=l(bo,"CODE",{});var Ap=o(La);or=r(Ap,"description"),Ap.forEach(a),nr=r(bo," attribute of the accuracy metric:"),bo.forEach(a),rl=u(e),m(st.$$.fragment,e),il=u(e),we=l(e,"P",{});var go=o(we);rr=r(go,"You can see that it describes how the metric works in theory. If you use this metric for your work, especially if it is an academic publication you want to reference it properly. For that you can look at the "),Ha=l(go,"CODE",{});var Pp=o(Ha);ir=r(Pp,"citation"),Pp.forEach(a),cr=r(go," attribute:"),go.forEach(a),cl=u(e),m(lt.$$.fragment,e),pl=u(e),Ct=l(e,"P",{});var Dp=o(Ct);pr=r(Dp,"Before we can apply a metric or other evaluation module to a use-case, we need to know what the input format of the metric is:"),Dp.forEach(a),ul=u(e),m(ot.$$.fragment,e),hl=u(e),m(Ee.$$.fragment,e),dl=u(e),X=l(e,"H2",{class:!0});var wo=o(X);$e=l(wo,"A",{id:!0,class:!0,href:!0});var Cp=o($e);Ua=l(Cp,"SPAN",{});var Op=o(Ua);m(nt.$$.fragment,Op),Op.forEach(a),Cp.forEach(a),ur=u(wo),Wa=l(wo,"SPAN",{});var Ip=o(Wa);hr=r(Ip,"Compute"),Ip.forEach(a),wo.forEach(a),ml=u(e),Ot=l(e,"P",{});var Sp=o(Ot);dr=r(Sp,"Now that we know how the evaluation module works and what should go in there we want to actually use it! When it comes to computing the actual score there are two main ways to do it:"),Sp.forEach(a),fl=u(e),je=l(e,"OL",{});var Eo=o(je);Fa=l(Eo,"LI",{});var Np=o(Fa);mr=r(Np,"All-in-one"),Np.forEach(a),fr=u(Eo),Ga=l(Eo,"LI",{});var Mp=o(Ga);vr=r(Mp,"Incremental"),Mp.forEach(a),Eo.forEach(a),vl=u(e),x=l(e,"P",{});var F=o(x);yr=r(F,"In the incremental approach the necessary inputs are added to the module with "),It=l(F,"A",{href:!0});var Bp=o(It);_r=r(Bp,"EvaluationModule.add()"),Bp.forEach(a),br=r(F," or "),St=l(F,"A",{href:!0});var Lp=o(St);gr=r(Lp,"EvaluationModule.add_batch()"),Lp.forEach(a),wr=r(F," and the score is calculated at the end with "),Nt=l(F,"A",{href:!0});var Hp=o(Nt);Er=r(Hp,"EvaluationModule.compute()"),Hp.forEach(a),$r=r(F,". Alternatively, one can pass all the inputs at once to "),Ra=l(F,"CODE",{});var Up=o(Ra);jr=r(Up,"compute()"),Up.forEach(a),kr=r(F,". Let\u2019s have a look at the two approaches."),F.forEach(a),yl=u(e),Z=l(e,"H3",{class:!0});var $o=o(Z);ke=l($o,"A",{id:!0,class:!0,href:!0});var Wp=o(ke);Ya=l(Wp,"SPAN",{});var Fp=o(Ya);m(rt.$$.fragment,Fp),Fp.forEach(a),Wp.forEach(a),xr=u($o),Mt=l($o,"SPAN",{});var Ec=o(Mt);Tr=r(Ec,"Using "),Va=l(Ec,"CODE",{});var Gp=o(Va);qr=r(Gp,"compute()"),Gp.forEach(a),Ec.forEach(a),$o.forEach(a),_l=u(e),D=l(e,"P",{});var Oe=o(D);Ar=r(Oe,"The simplest way to calculate the score of an evaluation module is by calling "),Ja=l(Oe,"CODE",{});var Rp=o(Ja);Pr=r(Rp,"compute()"),Rp.forEach(a),Dr=r(Oe," directly with the necessary inputs. Simply pass the inputs as seen in "),za=l(Oe,"CODE",{});var Yp=o(za);Cr=r(Yp,"features"),Yp.forEach(a),Or=r(Oe," to the "),Ka=l(Oe,"CODE",{});var Vp=o(Ka);Ir=r(Vp,"compute()"),Vp.forEach(a),Sr=r(Oe," method."),Oe.forEach(a),bl=u(e),m(it.$$.fragment,e),gl=u(e),B=l(e,"P",{});var Vt=o(B);Nr=r(Vt,"Evaluation modules return the results in a dictionary. However, in some instances you build up the predictions iteratively or in a distributed fashion in which case "),Qa=l(Vt,"CODE",{});var Jp=o(Qa);Mr=r(Jp,"add()"),Jp.forEach(a),Br=r(Vt," or "),Xa=l(Vt,"CODE",{});var zp=o(Xa);Lr=r(zp,"add_batch()"),zp.forEach(a),Hr=r(Vt," are useful."),Vt.forEach(a),wl=u(e),ee=l(e,"H3",{class:!0});var jo=o(ee);xe=l(jo,"A",{id:!0,class:!0,href:!0});var Kp=o(xe);Za=l(Kp,"SPAN",{});var Qp=o(Za);m(ct.$$.fragment,Qp),Qp.forEach(a),Kp.forEach(a),Ur=u(jo),Te=l(jo,"SPAN",{});var Bs=o(Te);Wr=r(Bs,"Using "),es=l(Bs,"CODE",{});var Xp=o(es);Fr=r(Xp,"add()"),Xp.forEach(a),Gr=r(Bs," and "),ts=l(Bs,"CODE",{});var Zp=o(ts);Rr=r(Zp,"add_batch()"),Zp.forEach(a),Bs.forEach(a),jo.forEach(a),El=u(e),T=l(e,"P",{});var G=o(T);Yr=r(G,"In many evaluation pipelines you build the predictions iteratively such as in a for-loop. In that case you could store the predictions in a list and at the end pass them to "),as=l(G,"CODE",{});var eu=o(as);Vr=r(eu,"compute()"),eu.forEach(a),Jr=r(G,". With "),ss=l(G,"CODE",{});var tu=o(ss);zr=r(tu,"add()"),tu.forEach(a),Kr=r(G," and "),ls=l(G,"CODE",{});var au=o(ls);Qr=r(au,"add_batch()"),au.forEach(a),Xr=r(G," you can circumvent the step of storing the predictions separately. If you are only creating single predictions at a time you can use "),os=l(G,"CODE",{});var su=o(os);Zr=r(su,"add()"),su.forEach(a),ei=r(G,":"),G.forEach(a),$l=u(e),m(pt.$$.fragment,e),jl=u(e),C=l(e,"P",{});var Ie=o(C);ti=r(Ie,"Once you have gathered all predictions you can call "),ns=l(Ie,"CODE",{});var lu=o(ns);ai=r(lu,"compute()"),lu.forEach(a),si=r(Ie," to compute the score based on all stored values. When getting predictions and references in batches you can use "),rs=l(Ie,"CODE",{});var ou=o(rs);li=r(ou,"add_batch()"),ou.forEach(a),oi=r(Ie," which adds a list elements for later processing. The rest works as with "),is=l(Ie,"CODE",{});var nu=o(is);ni=r(nu,"add()"),nu.forEach(a),ri=r(Ie,":"),Ie.forEach(a),kl=u(e),m(ut.$$.fragment,e),xl=u(e),Bt=l(e,"P",{});var ru=o(Bt);ii=r(ru,"This is especially useful when you need to get the predictions from your model in batches:"),ru.forEach(a),Tl=u(e),m(ht.$$.fragment,e),ql=u(e),te=l(e,"H3",{class:!0});var ko=o(te);qe=l(ko,"A",{id:!0,class:!0,href:!0});var iu=o(qe);cs=l(iu,"SPAN",{});var cu=o(cs);m(dt.$$.fragment,cu),cu.forEach(a),iu.forEach(a),ci=u(ko),ps=l(ko,"SPAN",{});var pu=o(ps);pi=r(pu,"Distributed evaluation"),pu.forEach(a),ko.forEach(a),Al=u(e),q=l(e,"P",{});var R=o(q);ui=r(R,"Computing metrics in a distributed environment can be tricky. Metric evaluation is executed in separate Python processes, or nodes, on different subsets of a dataset. Typically, when a metric score is additive ("),us=l(R,"CODE",{});var uu=o(us);hi=r(uu,"f(AuB) = f(A) + f(B)"),uu.forEach(a),di=r(R,"), you can use distributed reduce operations to gather the scores for each subset of the dataset. But when a metric is non-additive ("),hs=l(R,"CODE",{});var hu=o(hs);mi=r(hu,"f(AuB) \u2260 f(A) + f(B)"),hu.forEach(a),fi=r(R,"), it\u2019s not that simple. For example, you can\u2019t take the sum of the "),mt=l(R,"A",{href:!0,rel:!0});var du=o(mt);vi=r(du,"F1"),du.forEach(a),yi=r(R," scores of each data subset as your "),ds=l(R,"STRONG",{});var mu=o(ds);_i=r(mu,"final metric"),mu.forEach(a),bi=r(R,"."),R.forEach(a),Pl=u(e),Lt=l(e,"P",{});var fu=o(Lt);gi=r(fu,"A common way to overcome this issue is to fallback on single process evaluation. The metrics are evaluated on a single GPU, which becomes inefficient."),fu.forEach(a),Dl=u(e),L=l(e,"P",{});var Jt=o(L);wi=r(Jt,"\u{1F917} Evaluate solves this issue by only computing the final metric on the first node. The predictions and references are computed and provided to the metric separately for each node. These are temporarily stored in an Apache Arrow table, avoiding cluttering the GPU or CPU memory. When you are ready to "),ms=l(Jt,"CODE",{});var vu=o(ms);Ei=r(vu,"compute()"),vu.forEach(a),$i=r(Jt," the final metric, the first node is able to access the predictions and references stored on all the other nodes. Once it has gathered all the predictions and references, "),fs=l(Jt,"CODE",{});var yu=o(fs);ji=r(yu,"compute()"),yu.forEach(a),ki=r(Jt," will perform the final metric evaluation."),Jt.forEach(a),Cl=u(e),Ht=l(e,"P",{});var _u=o(Ht);xi=r(_u,"This solution allows \u{1F917} Evaluate to perform distributed predictions, which is important for evaluation speed in distributed settings. At the same time, you can also use complex non-additive metrics without wasting valuable GPU or CPU memory."),_u.forEach(a),Ol=u(e),ae=l(e,"H2",{class:!0});var xo=o(ae);Ae=l(xo,"A",{id:!0,class:!0,href:!0});var bu=o(Ae);vs=l(bu,"SPAN",{});var gu=o(vs);m(ft.$$.fragment,gu),gu.forEach(a),bu.forEach(a),Ti=u(xo),ys=l(xo,"SPAN",{});var wu=o(ys);qi=r(wu,"Save and push to the Hub"),wu.forEach(a),xo.forEach(a),Il=u(e),Pe=l(e,"P",{});var To=o(Pe);Ai=r(To,"Saving and sharing evaluation results is an important step. We provide the "),Ut=l(To,"A",{href:!0});var Eu=o(Ut);Pi=r(Eu,"evaluate.save()"),Eu.forEach(a),Di=r(To," function to easily save metrics results. You can either pass a specific filename or a directory. In the latter case, the results are saved in a file with an automatically created file name. Besides the directory or file name, the function takes any key-value pairs as inputs and stores them in a JSON file."),To.forEach(a),Sl=u(e),m(vt.$$.fragment,e),Nl=u(e),Wt=l(e,"P",{});var $u=o(Wt);Ci=r($u,"The content of the JSON file look like the following:"),$u.forEach(a),Ml=u(e),m(yt.$$.fragment,e),Bl=u(e),Ft=l(e,"P",{});var ju=o(Ft);Oi=r(ju,"In addition to the specified fields, it also contains useful system information for reproducing the results."),ju.forEach(a),Ll=u(e),De=l(e,"P",{});var qo=o(De);Ii=r(qo,"Besides storing the results locally, you should report them on the model\u2019s repository on the Hub. With the "),Gt=l(qo,"A",{href:!0});var ku=o(Gt);Si=r(ku,"evaluate.push_to_hub()"),ku.forEach(a),Ni=r(qo," function, you can easily report evaluation results to the model\u2019s repository:"),qo.forEach(a),Hl=u(e),m(_t.$$.fragment,e),Ul=u(e),se=l(e,"H2",{class:!0});var Ao=o(se);Ce=l(Ao,"A",{id:!0,class:!0,href:!0});var xu=o(Ce);_s=l(xu,"SPAN",{});var Tu=o(_s);m(bt.$$.fragment,Tu),Tu.forEach(a),xu.forEach(a),Mi=u(Ao),bs=l(Ao,"SPAN",{});var qu=o(bs);Bi=r(qu,"Evaluator"),qu.forEach(a),Ao.forEach(a),Wl=u(e),E=l(e,"P",{});var O=o(E);Li=r(O,"The "),Rt=l(O,"A",{href:!0});var Au=o(Rt);Hi=r(Au,"evaluate.evaluator()"),Au.forEach(a),Ui=r(O," provides automated evaluation and only requires a model, dataset, metric in contrast to the metrics in "),gs=l(O,"CODE",{});var Pu=o(gs);Wi=r(Pu,"EvaluationModule"),Pu.forEach(a),Fi=r(O,"s that require the model\u2019s predictions. As such it is easier to evaluate a model on a dataset with a given metric as the inference is handled internally. To make that possible it uses the "),ws=l(O,"CODE",{});var Du=o(ws);Gi=r(Du,"Pipeline"),Du.forEach(a),Ri=r(O," abstraction from "),Es=l(O,"CODE",{});var Cu=o(Es);Yi=r(Cu,"transformers"),Cu.forEach(a),Vi=r(O,". However, you can use your own framework as long as it follows the "),$s=l(O,"CODE",{});var Ou=o($s);Ji=r(Ou,"Pipeline"),Ou.forEach(a),zi=r(O," interface."),O.forEach(a),Fl=u(e),H=l(e,"P",{});var zt=o(H);Ki=r(zt,"To make an evaluation with the "),js=l(zt,"CODE",{});var Iu=o(js);Qi=r(Iu,"evaluator"),Iu.forEach(a),Xi=r(zt," let\u2019s load a "),ks=l(zt,"CODE",{});var Su=o(ks);Zi=r(Su,"transformers"),Su.forEach(a),ec=r(zt," pipeline (but you can pass your own custom inference class for any framework as long as it follows the pipeline call API) with an model trained on IMDb, the IMDb test split and the accuracy metric."),zt.forEach(a),Gl=u(e),m(gt.$$.fragment,e),Rl=u(e),U=l(e,"P",{});var Kt=o(U);tc=r(Kt,"Then you can create an evaluator for text classification and pass the three objects to the "),xs=l(Kt,"CODE",{});var Nu=o(xs);ac=r(Nu,"compute()"),Nu.forEach(a),sc=r(Kt," method. With the label mapping "),Ts=l(Kt,"CODE",{});var Mu=o(Ts);lc=r(Mu,"evaluate"),Mu.forEach(a),oc=r(Kt," provides a method to align the pipeline outputs with the label column in the dataset:"),Kt.forEach(a),Yl=u(e),m(wt.$$.fragment,e),Vl=u(e),W=l(e,"P",{});var Qt=o(W);nc=r(Qt,"Calculating the value of the metric alone is often not enough to know if a model performs significantly better than another one. With "),qs=l(Qt,"EM",{});var Bu=o(qs);rc=r(Bu,"bootstrapping"),Bu.forEach(a),ic=u(Qt),As=l(Qt,"CODE",{});var Lu=o(As);cc=r(Lu,"evaluate"),Lu.forEach(a),pc=r(Qt," computes confidence intervals and the standard error which helps estimate how stable a score is:"),Qt.forEach(a),Jl=u(e),m(Et.$$.fragment,e),zl=u(e),$=l(e,"P",{});var I=o($);uc=r(I,"The evaluator expects a "),Ps=l(I,"CODE",{});var Hu=o(Ps);hc=r(Hu,'"text"'),Hu.forEach(a),dc=r(I," and "),Ds=l(I,"CODE",{});var Uu=o(Ds);mc=r(Uu,'"label"'),Uu.forEach(a),fc=r(I," column for the data input. If your dataset differs you can provide the columns with the keywords "),Cs=l(I,"CODE",{});var Wu=o(Cs);vc=r(Wu,'input_column="text"'),Wu.forEach(a),yc=r(I," and "),Os=l(I,"CODE",{});var Fu=o(Os);_c=r(Fu,'label_column="label"'),Fu.forEach(a),bc=r(I,". Currently only "),Is=l(I,"CODE",{});var Gu=o(Is);gc=r(Gu,'"text-classification"'),Gu.forEach(a),wc=r(I," is supported with more tasks being added in the future."),I.forEach(a),this.h()},h(){h(g,"name","hf:doc:metadata"),h(g,"content",JSON.stringify(Zu)),h(P,"id","a-quick-tour"),h(P,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(P,"href","#a-quick-tour"),h(k,"class","relative group"),h(ne,"id","types-of-evaluations"),h(ne,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(ne,"href","#types-of-evaluations"),h(V,"class","relative group"),h(Ne,"href","https://huggingface.co/evaluate-metric"),h(Ne,"rel","nofollow"),h(Me,"href","https://huggingface.co/evaluate-comparison"),h(Me,"rel","nofollow"),h(Be,"href","https://huggingface.co/evaluate-measurement"),h(Be,"rel","nofollow"),h(xt,"href","/docs/evaluate/pr_137/en/package_reference/loading_methods#evaluate.load"),h(ue,"id","load"),h(ue,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(ue,"href","#load"),h(J,"class","relative group"),h(de,"id","community-modules"),h(de,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(de,"href","#community-modules"),h(z,"class","relative group"),h(At,"href","/docs/evaluate/main/en/creating_and_sharing"),h(fe,"id","list-available-modules"),h(fe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(fe,"href","#list-available-modules"),h(K,"class","relative group"),h(Pt,"href","/docs/evaluate/pr_137/en/package_reference/loading_methods#evaluate.list_evaluation_modules"),h(ye,"id","module-attributes"),h(ye,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(ye,"href","#module-attributes"),h(Q,"class","relative group"),h(Dt,"href","/docs/evaluate/pr_137/en/package_reference/main_classes#evaluate.EvaluationModuleInfo"),h($e,"id","compute"),h($e,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h($e,"href","#compute"),h(X,"class","relative group"),h(It,"href","/docs/evaluate/pr_137/en/package_reference/main_classes#evaluate.EvaluationModule.add"),h(St,"href","/docs/evaluate/pr_137/en/package_reference/main_classes#evaluate.EvaluationModule.add_batch"),h(Nt,"href","/docs/evaluate/pr_137/en/package_reference/main_classes#evaluate.EvaluationModule.compute"),h(ke,"id","using-compute"),h(ke,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(ke,"href","#using-compute"),h(Z,"class","relative group"),h(xe,"id","using-add-and-addbatch"),h(xe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(xe,"href","#using-add-and-addbatch"),h(ee,"class","relative group"),h(qe,"id","distributed-evaluation"),h(qe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(qe,"href","#distributed-evaluation"),h(te,"class","relative group"),h(mt,"href","https://huggingface.co/spaces/evaluate-metric/f1"),h(mt,"rel","nofollow"),h(Ae,"id","save-and-push-to-the-hub"),h(Ae,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Ae,"href","#save-and-push-to-the-hub"),h(ae,"class","relative group"),h(Ut,"href","/docs/evaluate/pr_137/en/package_reference/saving_methods#evaluate.save"),h(Gt,"href","/docs/evaluate/pr_137/en/package_reference/hub_methods#evaluate.push_to_hub"),h(Ce,"id","evaluator"),h(Ce,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Ce,"href","#evaluator"),h(se,"class","relative group"),h(Rt,"href","/docs/evaluate/pr_137/en/package_reference/evaluator_classes#evaluate.evaluator")},m(e,i){t(document.head,g),c(e,le,i),c(e,k,i),t(k,P),t(P,Y),f(A,Y,null),t(k,N),t(k,oe),t(oe,Po),c(e,Hs,i),c(e,jt,i),t(jt,Do),c(e,Us,i),c(e,V,i),t(V,ne),t(ne,Xt),f(Se,Xt,null),t(V,Co),t(V,Zt),t(Zt,Oo),c(e,Ws,i),c(e,kt,i),t(kt,Io),c(e,Fs,i),c(e,M,i),t(M,re),t(re,ea),t(ea,So),t(re,No),t(re,Ne),t(Ne,Mo),t(re,Bo),t(M,Lo),t(M,ie),t(ie,ta),t(ta,Ho),t(ie,Uo),t(ie,Me),t(Me,Wo),t(ie,Fo),t(M,Go),t(M,ce),t(ce,aa),t(aa,Ro),t(ce,Yo),t(ce,Be),t(Be,Vo),t(ce,Jo),c(e,Gs,i),c(e,pe,i),t(pe,zo),t(pe,xt),t(xt,Ko),t(pe,Qo),c(e,Rs,i),c(e,J,i),t(J,ue),t(ue,sa),f(Le,sa,null),t(J,Xo),t(J,la),t(la,Zo),c(e,Ys,i),c(e,he,i),t(he,en),t(he,oa),t(oa,tn),t(he,an),c(e,Vs,i),f(He,e,i),c(e,Js,i),c(e,Tt,i),t(Tt,sn),c(e,zs,i),f(Ue,e,i),c(e,Ks,i),c(e,z,i),t(z,de),t(de,na),f(We,na,null),t(z,ln),t(z,ra),t(ra,on),c(e,Qs,i),c(e,qt,i),t(qt,nn),c(e,Xs,i),f(Fe,e,i),c(e,Zs,i),c(e,me,i),t(me,rn),t(me,At),t(At,cn),t(me,pn),c(e,el,i),c(e,K,i),t(K,fe),t(fe,ia),f(Ge,ia,null),t(K,un),t(K,ca),t(ca,hn),c(e,tl,i),c(e,ve,i),t(ve,dn),t(ve,Pt),t(Pt,mn),t(ve,fn),c(e,al,i),f(Re,e,i),c(e,sl,i),c(e,Q,i),t(Q,ye),t(ye,pa),f(Ye,pa,null),t(Q,vn),t(Q,ua),t(ua,yn),c(e,ll,i),c(e,_e,i),t(_e,_n),t(_e,Dt),t(Dt,bn),t(_e,gn),c(e,ol,i),c(e,be,i),t(be,ha),t(ha,Ve),t(Ve,da),t(da,wn),t(Ve,En),t(Ve,ma),t(ma,$n),t(be,jn),t(be,b),t(b,Je),t(Je,fa),t(fa,va),t(va,kn),t(Je,xn),t(Je,ya),t(ya,Tn),t(b,qn),t(b,ze),t(ze,_a),t(_a,ba),t(ba,An),t(ze,Pn),t(ze,ga),t(ga,Dn),t(b,Cn),t(b,Ke),t(Ke,wa),t(wa,Ea),t(Ea,On),t(Ke,In),t(Ke,Qe),t(Qe,Sn),t(Qe,$a),t($a,Nn),t(Qe,Mn),t(b,Bn),t(b,Xe),t(Xe,ja),t(ja,ka),t(ka,Ln),t(Xe,Hn),t(Xe,xa),t(xa,Un),t(b,Wn),t(b,Ze),t(Ze,Ta),t(Ta,qa),t(qa,Fn),t(Ze,Gn),t(Ze,Aa),t(Aa,Rn),t(b,Yn),t(b,et),t(et,Pa),t(Pa,Da),t(Da,Vn),t(et,Jn),t(et,Ca),t(Ca,zn),t(b,Kn),t(b,tt),t(tt,Oa),t(Oa,Ia),t(Ia,Qn),t(tt,Xn),t(tt,Sa),t(Sa,Zn),t(b,er),t(b,at),t(at,Na),t(Na,Ma),t(Ma,tr),t(at,ar),t(at,Ba),t(Ba,sr),c(e,nl,i),c(e,ge,i),t(ge,lr),t(ge,La),t(La,or),t(ge,nr),c(e,rl,i),f(st,e,i),c(e,il,i),c(e,we,i),t(we,rr),t(we,Ha),t(Ha,ir),t(we,cr),c(e,cl,i),f(lt,e,i),c(e,pl,i),c(e,Ct,i),t(Ct,pr),c(e,ul,i),f(ot,e,i),c(e,hl,i),f(Ee,e,i),c(e,dl,i),c(e,X,i),t(X,$e),t($e,Ua),f(nt,Ua,null),t(X,ur),t(X,Wa),t(Wa,hr),c(e,ml,i),c(e,Ot,i),t(Ot,dr),c(e,fl,i),c(e,je,i),t(je,Fa),t(Fa,mr),t(je,fr),t(je,Ga),t(Ga,vr),c(e,vl,i),c(e,x,i),t(x,yr),t(x,It),t(It,_r),t(x,br),t(x,St),t(St,gr),t(x,wr),t(x,Nt),t(Nt,Er),t(x,$r),t(x,Ra),t(Ra,jr),t(x,kr),c(e,yl,i),c(e,Z,i),t(Z,ke),t(ke,Ya),f(rt,Ya,null),t(Z,xr),t(Z,Mt),t(Mt,Tr),t(Mt,Va),t(Va,qr),c(e,_l,i),c(e,D,i),t(D,Ar),t(D,Ja),t(Ja,Pr),t(D,Dr),t(D,za),t(za,Cr),t(D,Or),t(D,Ka),t(Ka,Ir),t(D,Sr),c(e,bl,i),f(it,e,i),c(e,gl,i),c(e,B,i),t(B,Nr),t(B,Qa),t(Qa,Mr),t(B,Br),t(B,Xa),t(Xa,Lr),t(B,Hr),c(e,wl,i),c(e,ee,i),t(ee,xe),t(xe,Za),f(ct,Za,null),t(ee,Ur),t(ee,Te),t(Te,Wr),t(Te,es),t(es,Fr),t(Te,Gr),t(Te,ts),t(ts,Rr),c(e,El,i),c(e,T,i),t(T,Yr),t(T,as),t(as,Vr),t(T,Jr),t(T,ss),t(ss,zr),t(T,Kr),t(T,ls),t(ls,Qr),t(T,Xr),t(T,os),t(os,Zr),t(T,ei),c(e,$l,i),f(pt,e,i),c(e,jl,i),c(e,C,i),t(C,ti),t(C,ns),t(ns,ai),t(C,si),t(C,rs),t(rs,li),t(C,oi),t(C,is),t(is,ni),t(C,ri),c(e,kl,i),f(ut,e,i),c(e,xl,i),c(e,Bt,i),t(Bt,ii),c(e,Tl,i),f(ht,e,i),c(e,ql,i),c(e,te,i),t(te,qe),t(qe,cs),f(dt,cs,null),t(te,ci),t(te,ps),t(ps,pi),c(e,Al,i),c(e,q,i),t(q,ui),t(q,us),t(us,hi),t(q,di),t(q,hs),t(hs,mi),t(q,fi),t(q,mt),t(mt,vi),t(q,yi),t(q,ds),t(ds,_i),t(q,bi),c(e,Pl,i),c(e,Lt,i),t(Lt,gi),c(e,Dl,i),c(e,L,i),t(L,wi),t(L,ms),t(ms,Ei),t(L,$i),t(L,fs),t(fs,ji),t(L,ki),c(e,Cl,i),c(e,Ht,i),t(Ht,xi),c(e,Ol,i),c(e,ae,i),t(ae,Ae),t(Ae,vs),f(ft,vs,null),t(ae,Ti),t(ae,ys),t(ys,qi),c(e,Il,i),c(e,Pe,i),t(Pe,Ai),t(Pe,Ut),t(Ut,Pi),t(Pe,Di),c(e,Sl,i),f(vt,e,i),c(e,Nl,i),c(e,Wt,i),t(Wt,Ci),c(e,Ml,i),f(yt,e,i),c(e,Bl,i),c(e,Ft,i),t(Ft,Oi),c(e,Ll,i),c(e,De,i),t(De,Ii),t(De,Gt),t(Gt,Si),t(De,Ni),c(e,Hl,i),f(_t,e,i),c(e,Ul,i),c(e,se,i),t(se,Ce),t(Ce,_s),f(bt,_s,null),t(se,Mi),t(se,bs),t(bs,Bi),c(e,Wl,i),c(e,E,i),t(E,Li),t(E,Rt),t(Rt,Hi),t(E,Ui),t(E,gs),t(gs,Wi),t(E,Fi),t(E,ws),t(ws,Gi),t(E,Ri),t(E,Es),t(Es,Yi),t(E,Vi),t(E,$s),t($s,Ji),t(E,zi),c(e,Fl,i),c(e,H,i),t(H,Ki),t(H,js),t(js,Qi),t(H,Xi),t(H,ks),t(ks,Zi),t(H,ec),c(e,Gl,i),f(gt,e,i),c(e,Rl,i),c(e,U,i),t(U,tc),t(U,xs),t(xs,ac),t(U,sc),t(U,Ts),t(Ts,lc),t(U,oc),c(e,Yl,i),f(wt,e,i),c(e,Vl,i),c(e,W,i),t(W,nc),t(W,qs),t(qs,rc),t(W,ic),t(W,As),t(As,cc),t(W,pc),c(e,Jl,i),f(Et,e,i),c(e,zl,i),c(e,$,i),t($,uc),t($,Ps),t(Ps,hc),t($,dc),t($,Ds),t(Ds,mc),t($,fc),t($,Cs),t(Cs,vc),t($,yc),t($,Os),t(Os,_c),t($,bc),t($,Is),t(Is,gc),t($,wc),Kl=!0},p(e,[i]){const $t={};i&2&&($t.$$scope={dirty:i,ctx:e}),Ee.$set($t)},i(e){Kl||(v(A.$$.fragment,e),v(Se.$$.fragment,e),v(Le.$$.fragment,e),v(He.$$.fragment,e),v(Ue.$$.fragment,e),v(We.$$.fragment,e),v(Fe.$$.fragment,e),v(Ge.$$.fragment,e),v(Re.$$.fragment,e),v(Ye.$$.fragment,e),v(st.$$.fragment,e),v(lt.$$.fragment,e),v(ot.$$.fragment,e),v(Ee.$$.fragment,e),v(nt.$$.fragment,e),v(rt.$$.fragment,e),v(it.$$.fragment,e),v(ct.$$.fragment,e),v(pt.$$.fragment,e),v(ut.$$.fragment,e),v(ht.$$.fragment,e),v(dt.$$.fragment,e),v(ft.$$.fragment,e),v(vt.$$.fragment,e),v(yt.$$.fragment,e),v(_t.$$.fragment,e),v(bt.$$.fragment,e),v(gt.$$.fragment,e),v(wt.$$.fragment,e),v(Et.$$.fragment,e),Kl=!0)},o(e){y(A.$$.fragment,e),y(Se.$$.fragment,e),y(Le.$$.fragment,e),y(He.$$.fragment,e),y(Ue.$$.fragment,e),y(We.$$.fragment,e),y(Fe.$$.fragment,e),y(Ge.$$.fragment,e),y(Re.$$.fragment,e),y(Ye.$$.fragment,e),y(st.$$.fragment,e),y(lt.$$.fragment,e),y(ot.$$.fragment,e),y(Ee.$$.fragment,e),y(nt.$$.fragment,e),y(rt.$$.fragment,e),y(it.$$.fragment,e),y(ct.$$.fragment,e),y(pt.$$.fragment,e),y(ut.$$.fragment,e),y(ht.$$.fragment,e),y(dt.$$.fragment,e),y(ft.$$.fragment,e),y(vt.$$.fragment,e),y(yt.$$.fragment,e),y(_t.$$.fragment,e),y(bt.$$.fragment,e),y(gt.$$.fragment,e),y(wt.$$.fragment,e),y(Et.$$.fragment,e),Kl=!1},d(e){a(g),e&&a(le),e&&a(k),_(A),e&&a(Hs),e&&a(jt),e&&a(Us),e&&a(V),_(Se),e&&a(Ws),e&&a(kt),e&&a(Fs),e&&a(M),e&&a(Gs),e&&a(pe),e&&a(Rs),e&&a(J),_(Le),e&&a(Ys),e&&a(he),e&&a(Vs),_(He,e),e&&a(Js),e&&a(Tt),e&&a(zs),_(Ue,e),e&&a(Ks),e&&a(z),_(We),e&&a(Qs),e&&a(qt),e&&a(Xs),_(Fe,e),e&&a(Zs),e&&a(me),e&&a(el),e&&a(K),_(Ge),e&&a(tl),e&&a(ve),e&&a(al),_(Re,e),e&&a(sl),e&&a(Q),_(Ye),e&&a(ll),e&&a(_e),e&&a(ol),e&&a(be),e&&a(nl),e&&a(ge),e&&a(rl),_(st,e),e&&a(il),e&&a(we),e&&a(cl),_(lt,e),e&&a(pl),e&&a(Ct),e&&a(ul),_(ot,e),e&&a(hl),_(Ee,e),e&&a(dl),e&&a(X),_(nt),e&&a(ml),e&&a(Ot),e&&a(fl),e&&a(je),e&&a(vl),e&&a(x),e&&a(yl),e&&a(Z),_(rt),e&&a(_l),e&&a(D),e&&a(bl),_(it,e),e&&a(gl),e&&a(B),e&&a(wl),e&&a(ee),_(ct),e&&a(El),e&&a(T),e&&a($l),_(pt,e),e&&a(jl),e&&a(C),e&&a(kl),_(ut,e),e&&a(xl),e&&a(Bt),e&&a(Tl),_(ht,e),e&&a(ql),e&&a(te),_(dt),e&&a(Al),e&&a(q),e&&a(Pl),e&&a(Lt),e&&a(Dl),e&&a(L),e&&a(Cl),e&&a(Ht),e&&a(Ol),e&&a(ae),_(ft),e&&a(Il),e&&a(Pe),e&&a(Sl),_(vt,e),e&&a(Nl),e&&a(Wt),e&&a(Ml),_(yt,e),e&&a(Bl),e&&a(Ft),e&&a(Ll),e&&a(De),e&&a(Hl),_(_t,e),e&&a(Ul),e&&a(se),_(bt),e&&a(Wl),e&&a(E),e&&a(Fl),e&&a(H),e&&a(Gl),_(gt,e),e&&a(Rl),e&&a(U),e&&a(Yl),_(wt,e),e&&a(Vl),e&&a(W),e&&a(Jl),_(Et,e),e&&a(zl),e&&a($)}}}const Zu={local:"a-quick-tour",sections:[{local:"types-of-evaluations",title:"Types of evaluations"},{local:"load",sections:[{local:"community-modules",title:"Community modules"},{local:"list-available-modules",title:"List available modules"}],title:"Load"},{local:"module-attributes",title:"Module attributes"},{local:"compute",sections:[{local:"using-compute",title:"Using `compute()`"},{local:"using-add-and-addbatch",title:"Using `add()` and `add_batch()`"},{local:"distributed-evaluation",title:"Distributed evaluation"}],title:"Compute"},{local:"save-and-push-to-the-hub",title:"Save and push to the Hub"},{local:"evaluator",title:"Evaluator"}],title:"A quick tour"};function eh(Ls){return zu(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class oh extends Ru{constructor(g){super();Yu(this,g,eh,Xu,Vu,{})}}export{oh as default,Zu as metadata};
