import{S as _l,i as gl,s as vl,e as a,k as c,w as _,t as n,M as wl,c as o,d as t,m as u,a as r,x as g,h as i,b as f,F as s,g as p,y as v,q as w,o as y,B as $,v as yl}from"../chunks/vendor-e830fc9c.js";import{C as I,T as Xo}from"../chunks/CodeBlock-7b28e4e9.js";import{I as ie}from"../chunks/IconCopyLink-af4c5c5b.js";function $l(B){let d,b;return{c(){d=a("p"),b=n("Metrics accepts various input formats (Python lists, NumPy arrays, PyTorch tensors, etc.) and converts them to an appropriate format for storage and computation.")},l(h){d=o(h,"P",{});var m=r(d);b=i(m,"Metrics accepts various input formats (Python lists, NumPy arrays, PyTorch tensors, etc.) and converts them to an appropriate format for storage and computation."),m.forEach(t)},m(h,m){p(h,d,m),s(d,b)},d(h){h&&t(d)}}}function bl(B){let d,b,h,m,L;return{c(){d=a("p"),b=n("Get jump started with our metric loading script "),h=a("a"),m=n("template"),L=n("!"),this.h()},l(E){d=o(E,"P",{});var q=r(d);b=i(q,"Get jump started with our metric loading script "),h=o(q,"A",{href:!0,rel:!0});var H=r(h);m=i(H,"template"),H.forEach(t),L=i(q,"!"),q.forEach(t),this.h()},h(){f(h,"href","https://github.com/huggingface/datasets/blob/master/templates/new_metric_script.py"),f(h,"rel","nofollow")},m(E,q){p(E,d,q),s(d,b),s(d,h),s(h,m),s(d,L)},d(E){E&&t(d)}}}function El(B){let d,b;return{c(){d=a("p"),b=n("If the files are stored locally, provide a dictionary of path(s) instead of URLs.")},l(h){d=o(h,"P",{});var m=r(d);b=i(m,"If the files are stored locally, provide a dictionary of path(s) instead of URLs."),m.forEach(t)},m(h,m){p(h,d,m),s(d,b)},d(h){h&&t(d)}}}function jl(B){let d,b,h,m,L,E,q,H,la,Xt,Ue,na,Zt,Re,ia,es,k,st,pa,ca,at,ua,fa,ot,da,ts,A,z,rt,pe,ha,lt,ma,ss,K,_a,nt,ga,va,as,V,it,P,pt,wa,ya,ct,$a,ba,ut,Ea,ja,qa,ft,C,dt,ka,Pa,ht,Ca,xa,mt,Ia,La,os,Y,Aa,_t,Sa,Ta,rs,ce,ls,Q,ns,S,W,gt,ue,Oa,vt,Da,is,F,Ma,wt,Na,Ua,ps,G,Ra,fe,Ba,Ha,cs,Be,yt,za,us,de,fs,he,$t,Ka,ds,me,hs,_e,T,Va,bt,Ya,Qa,Et,Wa,Fa,ms,ge,_s,He,gs,O,J,jt,ve,Ga,qt,Ja,vs,X,Xa,ze,Za,eo,ws,Z,to,we,so,ao,ys,ee,$s,D,te,kt,ye,oo,Pt,ro,bs,se,lo,Ct,no,io,Es,j,xt,Ke,It,po,co,uo,Lt,Ve,At,fo,ho,mo,St,Ye,Tt,_o,go,vo,Ot,Qe,Dt,wo,yo,js,We,$o,qs,$e,ks,M,ae,Mt,be,bo,Nt,Eo,Ps,x,jo,Ut,qo,ko,Ee,Po,Co,Cs,Fe,Rt,xo,xs,je,Is,oe,Ls,qe,Ge,Bt,Io,Lo,As,ke,Ss,N,re,Ht,Pe,Ao,zt,So,Ts,U,Kt,To,Oo,Ce,Do,Mo,Os,Je,xe,No,Vt,Uo,Ro,Ds,Ie,Ms,Le,Ae,Bo,Yt,Ho,zo,Ns,Se,Us,R,le,Qt,Te,Ko,Wt,Vo,Rs,Xe,Yo,Bs,Oe,Hs;return E=new ie({}),pe=new ie({}),ce=new I({props:{code:`import datasets
metric = evaluate.load('my_metric')
for model_input, gold_references in evaluation_dataset:
    model_predictions = model(model_inputs)
    metric.add_batch(predictions=model_predictions, references=gold_references)
final_score = metric.compute()`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> datasets
<span class="hljs-meta">&gt;&gt;&gt; </span>metric = evaluate.load(<span class="hljs-string">&#x27;my_metric&#x27;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">for</span> model_input, gold_references <span class="hljs-keyword">in</span> evaluation_dataset:
<span class="hljs-meta">... </span>    model_predictions = model(model_inputs)
<span class="hljs-meta">... </span>    metric.add_batch(predictions=model_predictions, references=gold_references)
<span class="hljs-meta">&gt;&gt;&gt; </span>final_score = metric.compute()`}}),Q=new Xo({props:{$$slots:{default:[$l]},$$scope:{ctx:B}}}),ue=new ie({}),de=new I({props:{code:`import datasets
metric = evaluate.load('sacrebleu')`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> datasets
<span class="hljs-meta">&gt;&gt;&gt; </span>metric = evaluate.load(<span class="hljs-string">&#x27;sacrebleu&#x27;</span>)`}}),me=new I({props:{code:`print(metric.inputs_description)
`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(metric.inputs_description)
Produces BLEU scores along <span class="hljs-keyword">with</span> its sufficient statistics
<span class="hljs-keyword">from</span> a source against one <span class="hljs-keyword">or</span> more references.

Args:
    predictions: The system stream (a sequence of segments).
    references: A <span class="hljs-built_in">list</span> of one <span class="hljs-keyword">or</span> more reference streams (each a sequence of segments).
    smooth_method: The smoothing method to use. (Default: <span class="hljs-string">&#x27;exp&#x27;</span>).
    smooth_value: The smoothing value. Only valid <span class="hljs-keyword">for</span> <span class="hljs-string">&#x27;floor&#x27;</span> <span class="hljs-keyword">and</span> <span class="hljs-string">&#x27;add-k&#x27;</span>. (Defaults: floor: <span class="hljs-number">0.1</span>, add-k: <span class="hljs-number">1</span>).
    tokenize: Tokenization method to use <span class="hljs-keyword">for</span> BLEU. If <span class="hljs-keyword">not</span> provided, defaults to <span class="hljs-string">&#x27;zh&#x27;</span> <span class="hljs-keyword">for</span> Chinese, <span class="hljs-string">&#x27;ja-mecab&#x27;</span> <span class="hljs-keyword">for</span> Japanese <span class="hljs-keyword">and</span> <span class="hljs-string">&#x27;13a&#x27;</span> (mteval) otherwise.
    lowercase: Lowercase the data. If <span class="hljs-literal">True</span>, enables case-insensitivity. (Default: <span class="hljs-literal">False</span>).
    force: Insist that your tokenized <span class="hljs-built_in">input</span> <span class="hljs-keyword">is</span> actually detokenized.
...`}}),ge=new I({props:{code:'score = metric.compute(smooth_method="floor", smooth_value=0.2)',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>score = metric.compute(smooth_method=<span class="hljs-string">&quot;floor&quot;</span>, smooth_value=<span class="hljs-number">0.2</span>)'}}),ve=new ie({}),ee=new Xo({props:{$$slots:{default:[bl]},$$scope:{ctx:B}}}),ye=new ie({}),$e=new I({props:{code:`class Squad(evaluate.Metric):
    def _info(self):
        return evaluate.EvaluationModuleInfo(
            description=_DESCRIPTION,
            citation=_CITATION,
            inputs_description=_KWARGS_DESCRIPTION,
            features=datasets.Features(
                {
                    "predictions": {"id": datasets.Value("string"), "prediction_text": datasets.Value("string")},
                    "references": {
                        "id": datasets.Value("string"),
                        "answers": datasets.features.Sequence(
                            {
                                "text": datasets.Value("string"),
                                "answer_start": datasets.Value("int32"),
                            }
                        ),
                    },
                }
            ),
            codebase_urls=["https://rajpurkar.github.io/SQuAD-explorer/"],
            reference_urls=["https://rajpurkar.github.io/SQuAD-explorer/"],
        )`,highlighted:`<span class="hljs-keyword">class</span> <span class="hljs-title class_">Squad</span>(evaluate.Metric):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">_info</span>(<span class="hljs-params">self</span>):
        <span class="hljs-keyword">return</span> evaluate.EvaluationModuleInfo(
            description=_DESCRIPTION,
            citation=_CITATION,
            inputs_description=_KWARGS_DESCRIPTION,
            features=datasets.Features(
                {
                    <span class="hljs-string">&quot;predictions&quot;</span>: {<span class="hljs-string">&quot;id&quot;</span>: datasets.Value(<span class="hljs-string">&quot;string&quot;</span>), <span class="hljs-string">&quot;prediction_text&quot;</span>: datasets.Value(<span class="hljs-string">&quot;string&quot;</span>)},
                    <span class="hljs-string">&quot;references&quot;</span>: {
                        <span class="hljs-string">&quot;id&quot;</span>: datasets.Value(<span class="hljs-string">&quot;string&quot;</span>),
                        <span class="hljs-string">&quot;answers&quot;</span>: datasets.features.<span class="hljs-type">Sequence</span>(
                            {
                                <span class="hljs-string">&quot;text&quot;</span>: datasets.Value(<span class="hljs-string">&quot;string&quot;</span>),
                                <span class="hljs-string">&quot;answer_start&quot;</span>: datasets.Value(<span class="hljs-string">&quot;int32&quot;</span>),
                            }
                        ),
                    },
                }
            ),
            codebase_urls=[<span class="hljs-string">&quot;https://rajpurkar.github.io/SQuAD-explorer/&quot;</span>],
            reference_urls=[<span class="hljs-string">&quot;https://rajpurkar.github.io/SQuAD-explorer/&quot;</span>],
        )`}}),be=new ie({}),je=new I({props:{code:`CHECKPOINT_URLS = {
    "bleurt-tiny-128": "https://storage.googleapis.com/bleurt-oss/bleurt-tiny-128.zip",
    "bleurt-tiny-512": "https://storage.googleapis.com/bleurt-oss/bleurt-tiny-512.zip",
    "bleurt-base-128": "https://storage.googleapis.com/bleurt-oss/bleurt-base-128.zip",
    "bleurt-base-512": "https://storage.googleapis.com/bleurt-oss/bleurt-base-512.zip",
    "bleurt-large-128": "https://storage.googleapis.com/bleurt-oss/bleurt-large-128.zip",
    "bleurt-large-512": "https://storage.googleapis.com/bleurt-oss/bleurt-large-512.zip",
}`,highlighted:`CHECKPOINT_URLS = {
    <span class="hljs-string">&quot;bleurt-tiny-128&quot;</span>: <span class="hljs-string">&quot;https://storage.googleapis.com/bleurt-oss/bleurt-tiny-128.zip&quot;</span>,
    <span class="hljs-string">&quot;bleurt-tiny-512&quot;</span>: <span class="hljs-string">&quot;https://storage.googleapis.com/bleurt-oss/bleurt-tiny-512.zip&quot;</span>,
    <span class="hljs-string">&quot;bleurt-base-128&quot;</span>: <span class="hljs-string">&quot;https://storage.googleapis.com/bleurt-oss/bleurt-base-128.zip&quot;</span>,
    <span class="hljs-string">&quot;bleurt-base-512&quot;</span>: <span class="hljs-string">&quot;https://storage.googleapis.com/bleurt-oss/bleurt-base-512.zip&quot;</span>,
    <span class="hljs-string">&quot;bleurt-large-128&quot;</span>: <span class="hljs-string">&quot;https://storage.googleapis.com/bleurt-oss/bleurt-large-128.zip&quot;</span>,
    <span class="hljs-string">&quot;bleurt-large-512&quot;</span>: <span class="hljs-string">&quot;https://storage.googleapis.com/bleurt-oss/bleurt-large-512.zip&quot;</span>,
}`}}),oe=new Xo({props:{$$slots:{default:[El]},$$scope:{ctx:B}}}),ke=new I({props:{code:`def _download_and_prepare(self, dl_manager):

    # check that config name specifies a valid BLEURT model
    if self.config_name == "default":
        logger.warning(
            "Using default BLEURT-Base checkpoint for sequence maximum length 128. "
            "You can use a bigger model for better results with e.g.: evaluate.load('bleurt', 'bleurt-large-512')."
        )
        self.config_name = "bleurt-base-128"
    if self.config_name not in CHECKPOINT_URLS.keys():
        raise KeyError(
            f"{self.config_name} model not found. You should supply the name of a model checkpoint for bleurt in {CHECKPOINT_URLS.keys()}"
        )

    # download the model checkpoint specified by self.config_name and set up the scorer
    model_path = dl_manager.download_and_extract(CHECKPOINT_URLS[self.config_name])
    self.scorer = score.BleurtScorer(os.path.join(model_path, self.config_name))`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">_download_and_prepare</span>(<span class="hljs-params">self, dl_manager</span>):

    <span class="hljs-comment"># check that config name specifies a valid BLEURT model</span>
    <span class="hljs-keyword">if</span> self.config_name == <span class="hljs-string">&quot;default&quot;</span>:
        logger.warning(
            <span class="hljs-string">&quot;Using default BLEURT-Base checkpoint for sequence maximum length 128. &quot;</span>
            <span class="hljs-string">&quot;You can use a bigger model for better results with e.g.: evaluate.load(&#x27;bleurt&#x27;, &#x27;bleurt-large-512&#x27;).&quot;</span>
        )
        self.config_name = <span class="hljs-string">&quot;bleurt-base-128&quot;</span>
    <span class="hljs-keyword">if</span> self.config_name <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> CHECKPOINT_URLS.keys():
        <span class="hljs-keyword">raise</span> KeyError(
            <span class="hljs-string">f&quot;<span class="hljs-subst">{self.config_name}</span> model not found. You should supply the name of a model checkpoint for bleurt in <span class="hljs-subst">{CHECKPOINT_URLS.keys()}</span>&quot;</span>
        )

    <span class="hljs-comment"># download the model checkpoint specified by self.config_name and set up the scorer</span>
    model_path = dl_manager.download_and_extract(CHECKPOINT_URLS[self.config_name])
    self.scorer = score.BleurtScorer(os.path.join(model_path, self.config_name))`}}),Pe=new ie({}),Ie=new I({props:{code:`def simple_accuracy(preds, labels):
    return (preds == labels).mean().item()

def acc_and_f1(preds, labels):
    acc = simple_accuracy(preds, labels)
    f1 = f1_score(y_true=labels, y_pred=preds).item()
    return {
        "accuracy": acc,
        "f1": f1,
    }

def pearson_and_spearman(preds, labels):
    pearson_corr = pearsonr(preds, labels)[0].item()
    spearman_corr = spearmanr(preds, labels)[0].item()
    return {
        "pearson": pearson_corr,
        "spearmanr": spearman_corr,
    }`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">simple_accuracy</span>(<span class="hljs-params">preds, labels</span>):
    <span class="hljs-keyword">return</span> (preds == labels).mean().item()

<span class="hljs-keyword">def</span> <span class="hljs-title function_">acc_and_f1</span>(<span class="hljs-params">preds, labels</span>):
    acc = simple_accuracy(preds, labels)
    f1 = f1_score(y_true=labels, y_pred=preds).item()
    <span class="hljs-keyword">return</span> {
        <span class="hljs-string">&quot;accuracy&quot;</span>: acc,
        <span class="hljs-string">&quot;f1&quot;</span>: f1,
    }

<span class="hljs-keyword">def</span> <span class="hljs-title function_">pearson_and_spearman</span>(<span class="hljs-params">preds, labels</span>):
    pearson_corr = pearsonr(preds, labels)[<span class="hljs-number">0</span>].item()
    spearman_corr = spearmanr(preds, labels)[<span class="hljs-number">0</span>].item()
    <span class="hljs-keyword">return</span> {
        <span class="hljs-string">&quot;pearson&quot;</span>: pearson_corr,
        <span class="hljs-string">&quot;spearmanr&quot;</span>: spearman_corr,
    }`}}),Se=new I({props:{code:`def _compute(self, predictions, references):
    if self.config_name == "cola":
        return {"matthews_correlation": matthews_corrcoef(references, predictions)}
    elif self.config_name == "stsb":
        return pearson_and_spearman(predictions, references)
    elif self.config_name in ["mrpc", "qqp"]:
        return acc_and_f1(predictions, references)
    elif self.config_name in ["sst2", "mnli", "mnli_mismatched", "mnli_matched", "qnli", "rte", "wnli", "hans"]:
        return {"accuracy": simple_accuracy(predictions, references)}
    else:
        raise KeyError(
            "You should supply a configuration name selected in "
            '["sst2", "mnli", "mnli_mismatched", "mnli_matched", '
            '"cola", "stsb", "mrpc", "qqp", "qnli", "rte", "wnli", "hans"]'
        )`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">_compute</span>(<span class="hljs-params">self, predictions, references</span>):
    <span class="hljs-keyword">if</span> self.config_name == <span class="hljs-string">&quot;cola&quot;</span>:
        <span class="hljs-keyword">return</span> {<span class="hljs-string">&quot;matthews_correlation&quot;</span>: matthews_corrcoef(references, predictions)}
    <span class="hljs-keyword">elif</span> self.config_name == <span class="hljs-string">&quot;stsb&quot;</span>:
        <span class="hljs-keyword">return</span> pearson_and_spearman(predictions, references)
    <span class="hljs-keyword">elif</span> self.config_name <span class="hljs-keyword">in</span> [<span class="hljs-string">&quot;mrpc&quot;</span>, <span class="hljs-string">&quot;qqp&quot;</span>]:
        <span class="hljs-keyword">return</span> acc_and_f1(predictions, references)
    <span class="hljs-keyword">elif</span> self.config_name <span class="hljs-keyword">in</span> [<span class="hljs-string">&quot;sst2&quot;</span>, <span class="hljs-string">&quot;mnli&quot;</span>, <span class="hljs-string">&quot;mnli_mismatched&quot;</span>, <span class="hljs-string">&quot;mnli_matched&quot;</span>, <span class="hljs-string">&quot;qnli&quot;</span>, <span class="hljs-string">&quot;rte&quot;</span>, <span class="hljs-string">&quot;wnli&quot;</span>, <span class="hljs-string">&quot;hans&quot;</span>]:
        <span class="hljs-keyword">return</span> {<span class="hljs-string">&quot;accuracy&quot;</span>: simple_accuracy(predictions, references)}
    <span class="hljs-keyword">else</span>:
        <span class="hljs-keyword">raise</span> KeyError(
            <span class="hljs-string">&quot;You should supply a configuration name selected in &quot;</span>
            <span class="hljs-string">&#x27;[&quot;sst2&quot;, &quot;mnli&quot;, &quot;mnli_mismatched&quot;, &quot;mnli_matched&quot;, &#x27;</span>
            <span class="hljs-string">&#x27;&quot;cola&quot;, &quot;stsb&quot;, &quot;mrpc&quot;, &quot;qqp&quot;, &quot;qnli&quot;, &quot;rte&quot;, &quot;wnli&quot;, &quot;hans&quot;]&#x27;</span>
        )`}}),Te=new ie({}),Oe=new I({props:{code:`from evaluate import load
metric = load('PATH/TO/MY/SCRIPT.py')`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> evaluate <span class="hljs-keyword">import</span> load
<span class="hljs-meta">&gt;&gt;&gt; </span>metric = load(<span class="hljs-string">&#x27;PATH/TO/MY/SCRIPT.py&#x27;</span>)`}}),{c(){d=a("meta"),b=c(),h=a("h1"),m=a("a"),L=a("span"),_(E.$$.fragment),q=c(),H=a("span"),la=n("Metrics"),Xt=c(),Ue=a("p"),na=n("Metrics are important for evaluating a model\u2019s predictions. In the tutorial, you learned how to compute a metric over an entire evaluation set. You have also seen how to load a metric."),Zt=c(),Re=a("p"),ia=n("This guide will show you how to:"),es=c(),k=a("ul"),st=a("li"),pa=n("Add predictions and references."),ca=c(),at=a("li"),ua=n("Compute metrics using different methods."),fa=c(),ot=a("li"),da=n("Write your own metric loading script."),ts=c(),A=a("h2"),z=a("a"),rt=a("span"),_(pe.$$.fragment),ha=c(),lt=a("span"),ma=n("Add predictions and references"),ss=c(),K=a("p"),_a=n("When you want to add model predictions and references to a "),nt=a("code"),ga=n("Metric"),va=n(" instance, you have two options:"),as=c(),V=a("ul"),it=a("li"),P=a("p"),pt=a("code"),wa=n("Metric.add"),ya=n(" adds a single "),ct=a("code"),$a=n("prediction"),ba=n(" and "),ut=a("code"),Ea=n("reference"),ja=n("."),qa=c(),ft=a("li"),C=a("p"),dt=a("code"),ka=n("Metric.add_batch"),Pa=n(" adds a batch of "),ht=a("code"),Ca=n("predictions"),xa=n(" and "),mt=a("code"),Ia=n("references"),La=n("."),os=c(),Y=a("p"),Aa=n("Use "),_t=a("code"),Sa=n("Metric.add_batch"),Ta=n(" by passing it your model predictions, and the references the model predictions should be evaluated against:"),rs=c(),_(ce.$$.fragment),ls=c(),_(Q.$$.fragment),ns=c(),S=a("h2"),W=a("a"),gt=a("span"),_(ue.$$.fragment),Oa=c(),vt=a("span"),Da=n("Compute scores"),is=c(),F=a("p"),Ma=n("The most straightforward way to calculate a metric is to call "),wt=a("code"),Na=n("Metric.compute"),Ua=n(". But some metrics have additional arguments that allow you to modify the metrics behavior."),ps=c(),G=a("p"),Ra=n("Let\u2019s load the "),fe=a("a"),Ba=n("SacreBLEU"),Ha=n(" metric, and compute it with a different smoothing method."),cs=c(),Be=a("ol"),yt=a("li"),za=n("Load the SacreBLEU metric:"),us=c(),_(de.$$.fragment),fs=c(),he=a("ol"),$t=a("li"),Ka=n("Inspect the different argument methods for computing the metric:"),ds=c(),_(me.$$.fragment),hs=c(),_e=a("ol"),T=a("li"),Va=n("Compute the metric with the "),bt=a("code"),Ya=n("floor"),Qa=n(" method, and a different "),Et=a("code"),Wa=n("smooth_value"),Fa=n(":"),ms=c(),_(ge.$$.fragment),_s=c(),He=a("a"),gs=c(),O=a("h2"),J=a("a"),jt=a("span"),_(ve.$$.fragment),Ga=c(),qt=a("span"),Ja=n("Custom metric loading script"),vs=c(),X=a("p"),Xa=n("Write a metric loading script to use your own custom metric (or one that is not on the Hub). Then you can load it as usual with "),ze=a("a"),Za=n("load()"),eo=n("."),ws=c(),Z=a("p"),to=n("To help you get started, open the "),we=a("a"),so=n("SQuAD metric loading script"),ao=n(" and follow along."),ys=c(),_(ee.$$.fragment),$s=c(),D=a("h3"),te=a("a"),kt=a("span"),_(ye.$$.fragment),oo=c(),Pt=a("span"),ro=n("Add metric attributes"),bs=c(),se=a("p"),lo=n("Start by adding some information about your metric in "),Ct=a("code"),no=n("Metric._info"),io=n(". The most important attributes you should specify are:"),Es=c(),j=a("ol"),xt=a("li"),Ke=a("p"),It=a("code"),po=n("EvaluationModuleInfo.description"),co=n(" provides a brief description about your metric."),uo=c(),Lt=a("li"),Ve=a("p"),At=a("code"),fo=n("EvaluationModuleInfo.citation"),ho=n(" contains a BibTex citation for the metric."),mo=c(),St=a("li"),Ye=a("p"),Tt=a("code"),_o=n("EvaluationModuleInfo.inputs_description"),go=n(" describes the expected inputs and outputs. It may also provide an example usage of the metric."),vo=c(),Ot=a("li"),Qe=a("p"),Dt=a("code"),wo=n("EvaluationModuleInfo.features"),yo=n(" defines the name and type of the predictions and references."),js=c(),We=a("p"),$o=n("After you\u2019ve filled out all these fields in the template, it should look like the following example from the SQuAD metric script:"),qs=c(),_($e.$$.fragment),ks=c(),M=a("h3"),ae=a("a"),Mt=a("span"),_(be.$$.fragment),bo=c(),Nt=a("span"),Eo=n("Download metric files"),Ps=c(),x=a("p"),jo=n("If your metric needs to download, or retrieve local files, you will need to use the "),Ut=a("code"),qo=n("Metric._download_and_prepare"),ko=n(" method. For this example, let\u2019s examine the "),Ee=a("a"),Po=n("BLEURT metric loading script"),Co=n("."),Cs=c(),Fe=a("ol"),Rt=a("li"),xo=n("Provide a dictionary of URLs that point to the metric files:"),xs=c(),_(je.$$.fragment),Is=c(),_(oe.$$.fragment),Ls=c(),qe=a("ol"),Ge=a("li"),Bt=a("code"),Io=n("Metric._download_and_prepare"),Lo=n(" will take the URLs and download the metric files specified:"),As=c(),_(ke.$$.fragment),Ss=c(),N=a("h3"),re=a("a"),Ht=a("span"),_(Pe.$$.fragment),Ao=c(),zt=a("span"),So=n("Compute score"),Ts=c(),U=a("p"),Kt=a("code"),To=n("DatasetBuilder._compute"),Oo=n(" provides the actual instructions for how to compute a metric given the predictions and references. Now let\u2019s take a look at the "),Ce=a("a"),Do=n("GLUE metric loading script"),Mo=n("."),Os=c(),Je=a("ol"),xe=a("li"),No=n("Provide the functions for "),Vt=a("code"),Uo=n("DatasetBuilder._compute"),Ro=n(" to calculate your metric:"),Ds=c(),_(Ie.$$.fragment),Ms=c(),Le=a("ol"),Ae=a("li"),Bo=n("Create "),Yt=a("code"),Ho=n("DatasetBuilder._compute"),zo=n(" with instructions for what metric to calculate for each configuration:"),Ns=c(),_(Se.$$.fragment),Us=c(),R=a("h3"),le=a("a"),Qt=a("span"),_(Te.$$.fragment),Ko=c(),Wt=a("span"),Vo=n("Test"),Rs=c(),Xe=a("p"),Yo=n("Once you\u2019re finished writing your metric loading script, try to load it locally:"),Bs=c(),_(Oe.$$.fragment),this.h()},l(e){const l=wl('[data-svelte="svelte-1phssyn"]',document.head);d=o(l,"META",{name:!0,content:!0}),l.forEach(t),b=u(e),h=o(e,"H1",{class:!0});var De=r(h);m=o(De,"A",{id:!0,class:!0,href:!0});var Ft=r(m);L=o(Ft,"SPAN",{});var Gt=r(L);g(E.$$.fragment,Gt),Gt.forEach(t),Ft.forEach(t),q=u(De),H=o(De,"SPAN",{});var Zo=r(H);la=i(Zo,"Metrics"),Zo.forEach(t),De.forEach(t),Xt=u(e),Ue=o(e,"P",{});var er=r(Ue);na=i(er,"Metrics are important for evaluating a model\u2019s predictions. In the tutorial, you learned how to compute a metric over an entire evaluation set. You have also seen how to load a metric."),er.forEach(t),Zt=u(e),Re=o(e,"P",{});var tr=r(Re);ia=i(tr,"This guide will show you how to:"),tr.forEach(t),es=u(e),k=o(e,"UL",{});var Ze=r(k);st=o(Ze,"LI",{});var sr=r(st);pa=i(sr,"Add predictions and references."),sr.forEach(t),ca=u(Ze),at=o(Ze,"LI",{});var ar=r(at);ua=i(ar,"Compute metrics using different methods."),ar.forEach(t),fa=u(Ze),ot=o(Ze,"LI",{});var or=r(ot);da=i(or,"Write your own metric loading script."),or.forEach(t),Ze.forEach(t),ts=u(e),A=o(e,"H2",{class:!0});var zs=r(A);z=o(zs,"A",{id:!0,class:!0,href:!0});var rr=r(z);rt=o(rr,"SPAN",{});var lr=r(rt);g(pe.$$.fragment,lr),lr.forEach(t),rr.forEach(t),ha=u(zs),lt=o(zs,"SPAN",{});var nr=r(lt);ma=i(nr,"Add predictions and references"),nr.forEach(t),zs.forEach(t),ss=u(e),K=o(e,"P",{});var Ks=r(K);_a=i(Ks,"When you want to add model predictions and references to a "),nt=o(Ks,"CODE",{});var ir=r(nt);ga=i(ir,"Metric"),ir.forEach(t),va=i(Ks," instance, you have two options:"),Ks.forEach(t),as=u(e),V=o(e,"UL",{});var Vs=r(V);it=o(Vs,"LI",{});var pr=r(it);P=o(pr,"P",{});var Me=r(P);pt=o(Me,"CODE",{});var cr=r(pt);wa=i(cr,"Metric.add"),cr.forEach(t),ya=i(Me," adds a single "),ct=o(Me,"CODE",{});var ur=r(ct);$a=i(ur,"prediction"),ur.forEach(t),ba=i(Me," and "),ut=o(Me,"CODE",{});var fr=r(ut);Ea=i(fr,"reference"),fr.forEach(t),ja=i(Me,"."),Me.forEach(t),pr.forEach(t),qa=u(Vs),ft=o(Vs,"LI",{});var dr=r(ft);C=o(dr,"P",{});var Ne=r(C);dt=o(Ne,"CODE",{});var hr=r(dt);ka=i(hr,"Metric.add_batch"),hr.forEach(t),Pa=i(Ne," adds a batch of "),ht=o(Ne,"CODE",{});var mr=r(ht);Ca=i(mr,"predictions"),mr.forEach(t),xa=i(Ne," and "),mt=o(Ne,"CODE",{});var _r=r(mt);Ia=i(_r,"references"),_r.forEach(t),La=i(Ne,"."),Ne.forEach(t),dr.forEach(t),Vs.forEach(t),os=u(e),Y=o(e,"P",{});var Ys=r(Y);Aa=i(Ys,"Use "),_t=o(Ys,"CODE",{});var gr=r(_t);Sa=i(gr,"Metric.add_batch"),gr.forEach(t),Ta=i(Ys," by passing it your model predictions, and the references the model predictions should be evaluated against:"),Ys.forEach(t),rs=u(e),g(ce.$$.fragment,e),ls=u(e),g(Q.$$.fragment,e),ns=u(e),S=o(e,"H2",{class:!0});var Qs=r(S);W=o(Qs,"A",{id:!0,class:!0,href:!0});var vr=r(W);gt=o(vr,"SPAN",{});var wr=r(gt);g(ue.$$.fragment,wr),wr.forEach(t),vr.forEach(t),Oa=u(Qs),vt=o(Qs,"SPAN",{});var yr=r(vt);Da=i(yr,"Compute scores"),yr.forEach(t),Qs.forEach(t),is=u(e),F=o(e,"P",{});var Ws=r(F);Ma=i(Ws,"The most straightforward way to calculate a metric is to call "),wt=o(Ws,"CODE",{});var $r=r(wt);Na=i($r,"Metric.compute"),$r.forEach(t),Ua=i(Ws,". But some metrics have additional arguments that allow you to modify the metrics behavior."),Ws.forEach(t),ps=u(e),G=o(e,"P",{});var Fs=r(G);Ra=i(Fs,"Let\u2019s load the "),fe=o(Fs,"A",{href:!0,rel:!0});var br=r(fe);Ba=i(br,"SacreBLEU"),br.forEach(t),Ha=i(Fs," metric, and compute it with a different smoothing method."),Fs.forEach(t),cs=u(e),Be=o(e,"OL",{});var Er=r(Be);yt=o(Er,"LI",{});var jr=r(yt);za=i(jr,"Load the SacreBLEU metric:"),jr.forEach(t),Er.forEach(t),us=u(e),g(de.$$.fragment,e),fs=u(e),he=o(e,"OL",{start:!0});var qr=r(he);$t=o(qr,"LI",{});var kr=r($t);Ka=i(kr,"Inspect the different argument methods for computing the metric:"),kr.forEach(t),qr.forEach(t),ds=u(e),g(me.$$.fragment,e),hs=u(e),_e=o(e,"OL",{start:!0});var Pr=r(_e);T=o(Pr,"LI",{});var et=r(T);Va=i(et,"Compute the metric with the "),bt=o(et,"CODE",{});var Cr=r(bt);Ya=i(Cr,"floor"),Cr.forEach(t),Qa=i(et," method, and a different "),Et=o(et,"CODE",{});var xr=r(Et);Wa=i(xr,"smooth_value"),xr.forEach(t),Fa=i(et,":"),et.forEach(t),Pr.forEach(t),ms=u(e),g(ge.$$.fragment,e),_s=u(e),He=o(e,"A",{id:!0}),r(He).forEach(t),gs=u(e),O=o(e,"H2",{class:!0});var Gs=r(O);J=o(Gs,"A",{id:!0,class:!0,href:!0});var Ir=r(J);jt=o(Ir,"SPAN",{});var Lr=r(jt);g(ve.$$.fragment,Lr),Lr.forEach(t),Ir.forEach(t),Ga=u(Gs),qt=o(Gs,"SPAN",{});var Ar=r(qt);Ja=i(Ar,"Custom metric loading script"),Ar.forEach(t),Gs.forEach(t),vs=u(e),X=o(e,"P",{});var Js=r(X);Xa=i(Js,"Write a metric loading script to use your own custom metric (or one that is not on the Hub). Then you can load it as usual with "),ze=o(Js,"A",{href:!0});var Sr=r(ze);Za=i(Sr,"load()"),Sr.forEach(t),eo=i(Js,"."),Js.forEach(t),ws=u(e),Z=o(e,"P",{});var Xs=r(Z);to=i(Xs,"To help you get started, open the "),we=o(Xs,"A",{href:!0,rel:!0});var Tr=r(we);so=i(Tr,"SQuAD metric loading script"),Tr.forEach(t),ao=i(Xs," and follow along."),Xs.forEach(t),ys=u(e),g(ee.$$.fragment,e),$s=u(e),D=o(e,"H3",{class:!0});var Zs=r(D);te=o(Zs,"A",{id:!0,class:!0,href:!0});var Or=r(te);kt=o(Or,"SPAN",{});var Dr=r(kt);g(ye.$$.fragment,Dr),Dr.forEach(t),Or.forEach(t),oo=u(Zs),Pt=o(Zs,"SPAN",{});var Mr=r(Pt);ro=i(Mr,"Add metric attributes"),Mr.forEach(t),Zs.forEach(t),bs=u(e),se=o(e,"P",{});var ea=r(se);lo=i(ea,"Start by adding some information about your metric in "),Ct=o(ea,"CODE",{});var Nr=r(Ct);no=i(Nr,"Metric._info"),Nr.forEach(t),io=i(ea,". The most important attributes you should specify are:"),ea.forEach(t),Es=u(e),j=o(e,"OL",{});var ne=r(j);xt=o(ne,"LI",{});var Ur=r(xt);Ke=o(Ur,"P",{});var Qo=r(Ke);It=o(Qo,"CODE",{});var Rr=r(It);po=i(Rr,"EvaluationModuleInfo.description"),Rr.forEach(t),co=i(Qo," provides a brief description about your metric."),Qo.forEach(t),Ur.forEach(t),uo=u(ne),Lt=o(ne,"LI",{});var Br=r(Lt);Ve=o(Br,"P",{});var Wo=r(Ve);At=o(Wo,"CODE",{});var Hr=r(At);fo=i(Hr,"EvaluationModuleInfo.citation"),Hr.forEach(t),ho=i(Wo," contains a BibTex citation for the metric."),Wo.forEach(t),Br.forEach(t),mo=u(ne),St=o(ne,"LI",{});var zr=r(St);Ye=o(zr,"P",{});var Fo=r(Ye);Tt=o(Fo,"CODE",{});var Kr=r(Tt);_o=i(Kr,"EvaluationModuleInfo.inputs_description"),Kr.forEach(t),go=i(Fo," describes the expected inputs and outputs. It may also provide an example usage of the metric."),Fo.forEach(t),zr.forEach(t),vo=u(ne),Ot=o(ne,"LI",{});var Vr=r(Ot);Qe=o(Vr,"P",{});var Go=r(Qe);Dt=o(Go,"CODE",{});var Yr=r(Dt);wo=i(Yr,"EvaluationModuleInfo.features"),Yr.forEach(t),yo=i(Go," defines the name and type of the predictions and references."),Go.forEach(t),Vr.forEach(t),ne.forEach(t),js=u(e),We=o(e,"P",{});var Qr=r(We);$o=i(Qr,"After you\u2019ve filled out all these fields in the template, it should look like the following example from the SQuAD metric script:"),Qr.forEach(t),qs=u(e),g($e.$$.fragment,e),ks=u(e),M=o(e,"H3",{class:!0});var ta=r(M);ae=o(ta,"A",{id:!0,class:!0,href:!0});var Wr=r(ae);Mt=o(Wr,"SPAN",{});var Fr=r(Mt);g(be.$$.fragment,Fr),Fr.forEach(t),Wr.forEach(t),bo=u(ta),Nt=o(ta,"SPAN",{});var Gr=r(Nt);Eo=i(Gr,"Download metric files"),Gr.forEach(t),ta.forEach(t),Ps=u(e),x=o(e,"P",{});var tt=r(x);jo=i(tt,"If your metric needs to download, or retrieve local files, you will need to use the "),Ut=o(tt,"CODE",{});var Jr=r(Ut);qo=i(Jr,"Metric._download_and_prepare"),Jr.forEach(t),ko=i(tt," method. For this example, let\u2019s examine the "),Ee=o(tt,"A",{href:!0,rel:!0});var Xr=r(Ee);Po=i(Xr,"BLEURT metric loading script"),Xr.forEach(t),Co=i(tt,"."),tt.forEach(t),Cs=u(e),Fe=o(e,"OL",{});var Zr=r(Fe);Rt=o(Zr,"LI",{});var el=r(Rt);xo=i(el,"Provide a dictionary of URLs that point to the metric files:"),el.forEach(t),Zr.forEach(t),xs=u(e),g(je.$$.fragment,e),Is=u(e),g(oe.$$.fragment,e),Ls=u(e),qe=o(e,"OL",{start:!0});var tl=r(qe);Ge=o(tl,"LI",{});var Jo=r(Ge);Bt=o(Jo,"CODE",{});var sl=r(Bt);Io=i(sl,"Metric._download_and_prepare"),sl.forEach(t),Lo=i(Jo," will take the URLs and download the metric files specified:"),Jo.forEach(t),tl.forEach(t),As=u(e),g(ke.$$.fragment,e),Ss=u(e),N=o(e,"H3",{class:!0});var sa=r(N);re=o(sa,"A",{id:!0,class:!0,href:!0});var al=r(re);Ht=o(al,"SPAN",{});var ol=r(Ht);g(Pe.$$.fragment,ol),ol.forEach(t),al.forEach(t),Ao=u(sa),zt=o(sa,"SPAN",{});var rl=r(zt);So=i(rl,"Compute score"),rl.forEach(t),sa.forEach(t),Ts=u(e),U=o(e,"P",{});var Jt=r(U);Kt=o(Jt,"CODE",{});var ll=r(Kt);To=i(ll,"DatasetBuilder._compute"),ll.forEach(t),Oo=i(Jt," provides the actual instructions for how to compute a metric given the predictions and references. Now let\u2019s take a look at the "),Ce=o(Jt,"A",{href:!0,rel:!0});var nl=r(Ce);Do=i(nl,"GLUE metric loading script"),nl.forEach(t),Mo=i(Jt,"."),Jt.forEach(t),Os=u(e),Je=o(e,"OL",{});var il=r(Je);xe=o(il,"LI",{});var aa=r(xe);No=i(aa,"Provide the functions for "),Vt=o(aa,"CODE",{});var pl=r(Vt);Uo=i(pl,"DatasetBuilder._compute"),pl.forEach(t),Ro=i(aa," to calculate your metric:"),aa.forEach(t),il.forEach(t),Ds=u(e),g(Ie.$$.fragment,e),Ms=u(e),Le=o(e,"OL",{start:!0});var cl=r(Le);Ae=o(cl,"LI",{});var oa=r(Ae);Bo=i(oa,"Create "),Yt=o(oa,"CODE",{});var ul=r(Yt);Ho=i(ul,"DatasetBuilder._compute"),ul.forEach(t),zo=i(oa," with instructions for what metric to calculate for each configuration:"),oa.forEach(t),cl.forEach(t),Ns=u(e),g(Se.$$.fragment,e),Us=u(e),R=o(e,"H3",{class:!0});var ra=r(R);le=o(ra,"A",{id:!0,class:!0,href:!0});var fl=r(le);Qt=o(fl,"SPAN",{});var dl=r(Qt);g(Te.$$.fragment,dl),dl.forEach(t),fl.forEach(t),Ko=u(ra),Wt=o(ra,"SPAN",{});var hl=r(Wt);Vo=i(hl,"Test"),hl.forEach(t),ra.forEach(t),Rs=u(e),Xe=o(e,"P",{});var ml=r(Xe);Yo=i(ml,"Once you\u2019re finished writing your metric loading script, try to load it locally:"),ml.forEach(t),Bs=u(e),g(Oe.$$.fragment,e),this.h()},h(){f(d,"name","hf:doc:metadata"),f(d,"content",JSON.stringify(ql)),f(m,"id","metrics"),f(m,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(m,"href","#metrics"),f(h,"class","relative group"),f(z,"id","add-predictions-and-references"),f(z,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(z,"href","#add-predictions-and-references"),f(A,"class","relative group"),f(W,"id","compute-scores"),f(W,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(W,"href","#compute-scores"),f(S,"class","relative group"),f(fe,"href","https://huggingface.co/metrics/sacrebleu"),f(fe,"rel","nofollow"),f(he,"start","2"),f(_e,"start","3"),f(He,"id","metric_script"),f(J,"id","custom-metric-loading-script"),f(J,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(J,"href","#custom-metric-loading-script"),f(O,"class","relative group"),f(ze,"href","/docs/evaluate/pr_50/en/package_reference/loading_methods#evaluate.load"),f(we,"href","https://github.com/huggingface/datasets/blob/master/metrics/squad/squad.py"),f(we,"rel","nofollow"),f(te,"id","add-metric-attributes"),f(te,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(te,"href","#add-metric-attributes"),f(D,"class","relative group"),f(ae,"id","download-metric-files"),f(ae,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(ae,"href","#download-metric-files"),f(M,"class","relative group"),f(Ee,"href","https://github.com/huggingface/datasets/blob/master/metrics/bleurt/bleurt.py"),f(Ee,"rel","nofollow"),f(qe,"start","2"),f(re,"id","compute-score"),f(re,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(re,"href","#compute-score"),f(N,"class","relative group"),f(Ce,"href","https://github.com/huggingface/datasets/blob/master/metrics/glue/glue.py"),f(Ce,"rel","nofollow"),f(Le,"start","2"),f(le,"id","test"),f(le,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(le,"href","#test"),f(R,"class","relative group")},m(e,l){s(document.head,d),p(e,b,l),p(e,h,l),s(h,m),s(m,L),v(E,L,null),s(h,q),s(h,H),s(H,la),p(e,Xt,l),p(e,Ue,l),s(Ue,na),p(e,Zt,l),p(e,Re,l),s(Re,ia),p(e,es,l),p(e,k,l),s(k,st),s(st,pa),s(k,ca),s(k,at),s(at,ua),s(k,fa),s(k,ot),s(ot,da),p(e,ts,l),p(e,A,l),s(A,z),s(z,rt),v(pe,rt,null),s(A,ha),s(A,lt),s(lt,ma),p(e,ss,l),p(e,K,l),s(K,_a),s(K,nt),s(nt,ga),s(K,va),p(e,as,l),p(e,V,l),s(V,it),s(it,P),s(P,pt),s(pt,wa),s(P,ya),s(P,ct),s(ct,$a),s(P,ba),s(P,ut),s(ut,Ea),s(P,ja),s(V,qa),s(V,ft),s(ft,C),s(C,dt),s(dt,ka),s(C,Pa),s(C,ht),s(ht,Ca),s(C,xa),s(C,mt),s(mt,Ia),s(C,La),p(e,os,l),p(e,Y,l),s(Y,Aa),s(Y,_t),s(_t,Sa),s(Y,Ta),p(e,rs,l),v(ce,e,l),p(e,ls,l),v(Q,e,l),p(e,ns,l),p(e,S,l),s(S,W),s(W,gt),v(ue,gt,null),s(S,Oa),s(S,vt),s(vt,Da),p(e,is,l),p(e,F,l),s(F,Ma),s(F,wt),s(wt,Na),s(F,Ua),p(e,ps,l),p(e,G,l),s(G,Ra),s(G,fe),s(fe,Ba),s(G,Ha),p(e,cs,l),p(e,Be,l),s(Be,yt),s(yt,za),p(e,us,l),v(de,e,l),p(e,fs,l),p(e,he,l),s(he,$t),s($t,Ka),p(e,ds,l),v(me,e,l),p(e,hs,l),p(e,_e,l),s(_e,T),s(T,Va),s(T,bt),s(bt,Ya),s(T,Qa),s(T,Et),s(Et,Wa),s(T,Fa),p(e,ms,l),v(ge,e,l),p(e,_s,l),p(e,He,l),p(e,gs,l),p(e,O,l),s(O,J),s(J,jt),v(ve,jt,null),s(O,Ga),s(O,qt),s(qt,Ja),p(e,vs,l),p(e,X,l),s(X,Xa),s(X,ze),s(ze,Za),s(X,eo),p(e,ws,l),p(e,Z,l),s(Z,to),s(Z,we),s(we,so),s(Z,ao),p(e,ys,l),v(ee,e,l),p(e,$s,l),p(e,D,l),s(D,te),s(te,kt),v(ye,kt,null),s(D,oo),s(D,Pt),s(Pt,ro),p(e,bs,l),p(e,se,l),s(se,lo),s(se,Ct),s(Ct,no),s(se,io),p(e,Es,l),p(e,j,l),s(j,xt),s(xt,Ke),s(Ke,It),s(It,po),s(Ke,co),s(j,uo),s(j,Lt),s(Lt,Ve),s(Ve,At),s(At,fo),s(Ve,ho),s(j,mo),s(j,St),s(St,Ye),s(Ye,Tt),s(Tt,_o),s(Ye,go),s(j,vo),s(j,Ot),s(Ot,Qe),s(Qe,Dt),s(Dt,wo),s(Qe,yo),p(e,js,l),p(e,We,l),s(We,$o),p(e,qs,l),v($e,e,l),p(e,ks,l),p(e,M,l),s(M,ae),s(ae,Mt),v(be,Mt,null),s(M,bo),s(M,Nt),s(Nt,Eo),p(e,Ps,l),p(e,x,l),s(x,jo),s(x,Ut),s(Ut,qo),s(x,ko),s(x,Ee),s(Ee,Po),s(x,Co),p(e,Cs,l),p(e,Fe,l),s(Fe,Rt),s(Rt,xo),p(e,xs,l),v(je,e,l),p(e,Is,l),v(oe,e,l),p(e,Ls,l),p(e,qe,l),s(qe,Ge),s(Ge,Bt),s(Bt,Io),s(Ge,Lo),p(e,As,l),v(ke,e,l),p(e,Ss,l),p(e,N,l),s(N,re),s(re,Ht),v(Pe,Ht,null),s(N,Ao),s(N,zt),s(zt,So),p(e,Ts,l),p(e,U,l),s(U,Kt),s(Kt,To),s(U,Oo),s(U,Ce),s(Ce,Do),s(U,Mo),p(e,Os,l),p(e,Je,l),s(Je,xe),s(xe,No),s(xe,Vt),s(Vt,Uo),s(xe,Ro),p(e,Ds,l),v(Ie,e,l),p(e,Ms,l),p(e,Le,l),s(Le,Ae),s(Ae,Bo),s(Ae,Yt),s(Yt,Ho),s(Ae,zo),p(e,Ns,l),v(Se,e,l),p(e,Us,l),p(e,R,l),s(R,le),s(le,Qt),v(Te,Qt,null),s(R,Ko),s(R,Wt),s(Wt,Vo),p(e,Rs,l),p(e,Xe,l),s(Xe,Yo),p(e,Bs,l),v(Oe,e,l),Hs=!0},p(e,[l]){const De={};l&2&&(De.$$scope={dirty:l,ctx:e}),Q.$set(De);const Ft={};l&2&&(Ft.$$scope={dirty:l,ctx:e}),ee.$set(Ft);const Gt={};l&2&&(Gt.$$scope={dirty:l,ctx:e}),oe.$set(Gt)},i(e){Hs||(w(E.$$.fragment,e),w(pe.$$.fragment,e),w(ce.$$.fragment,e),w(Q.$$.fragment,e),w(ue.$$.fragment,e),w(de.$$.fragment,e),w(me.$$.fragment,e),w(ge.$$.fragment,e),w(ve.$$.fragment,e),w(ee.$$.fragment,e),w(ye.$$.fragment,e),w($e.$$.fragment,e),w(be.$$.fragment,e),w(je.$$.fragment,e),w(oe.$$.fragment,e),w(ke.$$.fragment,e),w(Pe.$$.fragment,e),w(Ie.$$.fragment,e),w(Se.$$.fragment,e),w(Te.$$.fragment,e),w(Oe.$$.fragment,e),Hs=!0)},o(e){y(E.$$.fragment,e),y(pe.$$.fragment,e),y(ce.$$.fragment,e),y(Q.$$.fragment,e),y(ue.$$.fragment,e),y(de.$$.fragment,e),y(me.$$.fragment,e),y(ge.$$.fragment,e),y(ve.$$.fragment,e),y(ee.$$.fragment,e),y(ye.$$.fragment,e),y($e.$$.fragment,e),y(be.$$.fragment,e),y(je.$$.fragment,e),y(oe.$$.fragment,e),y(ke.$$.fragment,e),y(Pe.$$.fragment,e),y(Ie.$$.fragment,e),y(Se.$$.fragment,e),y(Te.$$.fragment,e),y(Oe.$$.fragment,e),Hs=!1},d(e){t(d),e&&t(b),e&&t(h),$(E),e&&t(Xt),e&&t(Ue),e&&t(Zt),e&&t(Re),e&&t(es),e&&t(k),e&&t(ts),e&&t(A),$(pe),e&&t(ss),e&&t(K),e&&t(as),e&&t(V),e&&t(os),e&&t(Y),e&&t(rs),$(ce,e),e&&t(ls),$(Q,e),e&&t(ns),e&&t(S),$(ue),e&&t(is),e&&t(F),e&&t(ps),e&&t(G),e&&t(cs),e&&t(Be),e&&t(us),$(de,e),e&&t(fs),e&&t(he),e&&t(ds),$(me,e),e&&t(hs),e&&t(_e),e&&t(ms),$(ge,e),e&&t(_s),e&&t(He),e&&t(gs),e&&t(O),$(ve),e&&t(vs),e&&t(X),e&&t(ws),e&&t(Z),e&&t(ys),$(ee,e),e&&t($s),e&&t(D),$(ye),e&&t(bs),e&&t(se),e&&t(Es),e&&t(j),e&&t(js),e&&t(We),e&&t(qs),$($e,e),e&&t(ks),e&&t(M),$(be),e&&t(Ps),e&&t(x),e&&t(Cs),e&&t(Fe),e&&t(xs),$(je,e),e&&t(Is),$(oe,e),e&&t(Ls),e&&t(qe),e&&t(As),$(ke,e),e&&t(Ss),e&&t(N),$(Pe),e&&t(Ts),e&&t(U),e&&t(Os),e&&t(Je),e&&t(Ds),$(Ie,e),e&&t(Ms),e&&t(Le),e&&t(Ns),$(Se,e),e&&t(Us),e&&t(R),$(Te),e&&t(Rs),e&&t(Xe),e&&t(Bs),$(Oe,e)}}}const ql={local:"metrics",sections:[{local:"add-predictions-and-references",title:"Add predictions and references"},{local:"compute-scores",title:"Compute scores"},{local:"custom-metric-loading-script",sections:[{local:"add-metric-attributes",title:"Add metric attributes"},{local:"download-metric-files",title:"Download metric files"},{local:"compute-score",title:"Compute score"},{local:"test",title:"Test"}],title:"Custom metric loading script"}],title:"Metrics"};function kl(B){return yl(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Il extends _l{constructor(d){super();gl(this,d,kl,jl,vl,{})}}export{Il as default,ql as metadata};
