import{S as Gt,i as Jt,s as Kt,e as s,k as c,w as y,t as l,M as Qt,c as n,d as t,m as d,a as r,x as $,h as i,b as v,G as e,g as f,y as w,q as E,o as x,B as q,v as Xt,L as vt}from"../../chunks/vendor-hf-doc-builder.js";import{D as F}from"../../chunks/Docstring-hf-doc-builder.js";import{C as _t}from"../../chunks/CodeBlock-hf-doc-builder.js";import{I as Wt}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{E as ht}from"../../chunks/ExampleCodeBlock-hf-doc-builder.js";function Yt(O){let u,T,g,p,_;return p=new _t({props:{code:`from evaluate import evaluator
# Sentiment analysis evaluator
evaluator("sentiment-analysis")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> evaluate <span class="hljs-keyword">import</span> evaluator
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Sentiment analysis evaluator</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>evaluator(<span class="hljs-string">&quot;sentiment-analysis&quot;</span>)`}}),{c(){u=s("p"),T=l("Examples:"),g=c(),y(p.$$.fragment)},l(o){u=n(o,"P",{});var h=r(u);T=i(h,"Examples:"),h.forEach(t),g=d(o),$(p.$$.fragment,o)},m(o,h){f(o,u,h),e(u,T),f(o,g,h),w(p,o,h),_=!0},p:vt,i(o){_||(E(p.$$.fragment,o),_=!0)},o(o){x(p.$$.fragment,o),_=!1},d(o){o&&t(u),o&&t(g),q(p,o)}}}function Zt(O){let u,T,g,p,_;return p=new _t({props:{code:`from evaluate import evaluator
from datasets import Dataset, load_dataset
e = evaluator("text-classification")
data =  Dataset.from_dict(load_dataset("imdb")["test"][:2])
results = e.compute(
    model_or_pipeline="huggingface/prunebert-base-uncased-6-finepruned-w-distil-mnli",
    data=data,
    metric="accuracy",
    input_column="text",
    label_column="label",
    label_mapping={"LABEL_0": 0.0, "LABEL_1": 1.0},
    strategy="bootstrap",
    n_resamples=10,
    random_state=0
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> evaluate <span class="hljs-keyword">import</span> evaluator
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset, load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>e = evaluator(<span class="hljs-string">&quot;text-classification&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>data =  Dataset.from_dict(load_dataset(<span class="hljs-string">&quot;imdb&quot;</span>)[<span class="hljs-string">&quot;test&quot;</span>][:<span class="hljs-number">2</span>])
<span class="hljs-meta">&gt;&gt;&gt; </span>results = e.compute(
<span class="hljs-meta">&gt;&gt;&gt; </span>    model_or_pipeline=<span class="hljs-string">&quot;huggingface/prunebert-base-uncased-6-finepruned-w-distil-mnli&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    data=data,
<span class="hljs-meta">&gt;&gt;&gt; </span>    metric=<span class="hljs-string">&quot;accuracy&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    input_column=<span class="hljs-string">&quot;text&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    label_column=<span class="hljs-string">&quot;label&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    label_mapping={<span class="hljs-string">&quot;LABEL_0&quot;</span>: <span class="hljs-number">0.0</span>, <span class="hljs-string">&quot;LABEL_1&quot;</span>: <span class="hljs-number">1.0</span>},
<span class="hljs-meta">&gt;&gt;&gt; </span>    strategy=<span class="hljs-string">&quot;bootstrap&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    n_resamples=<span class="hljs-number">10</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    random_state=<span class="hljs-number">0</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>)`}}),{c(){u=s("p"),T=l("Examples:"),g=c(),y(p.$$.fragment)},l(o){u=n(o,"P",{});var h=r(u);T=i(h,"Examples:"),h.forEach(t),g=d(o),$(p.$$.fragment,o)},m(o,h){f(o,u,h),e(u,T),f(o,g,h),w(p,o,h),_=!0},p:vt,i(o){_||(E(p.$$.fragment,o),_=!0)},o(o){x(p.$$.fragment,o),_=!1},d(o){o&&t(u),o&&t(g),q(p,o)}}}function eo(O){let u,T,g,p,_;return p=new _t({props:{code:`from evaluate import evaluator
from datasets import Dataset, load_dataset
e = evaluator("image-classification")
data =  Dataset.from_dict(load_dataset("beans")["test"][:2])
results = e.compute(
    model_or_pipeline="nateraw/vit-base-beans",
    data=data,
    metric="accuracy",
    input_column="image",
    label_column="labels",
    label_mapping={'angular_leaf_spot': 0, 'bean_rust': 1, 'healthy': 2},
    strategy="bootstrap",
    n_resamples=10,
    random_state=0
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> evaluate <span class="hljs-keyword">import</span> evaluator
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset, load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>e = evaluator(<span class="hljs-string">&quot;image-classification&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>data =  Dataset.from_dict(load_dataset(<span class="hljs-string">&quot;beans&quot;</span>)[<span class="hljs-string">&quot;test&quot;</span>][:<span class="hljs-number">2</span>])
<span class="hljs-meta">&gt;&gt;&gt; </span>results = e.compute(
<span class="hljs-meta">&gt;&gt;&gt; </span>    model_or_pipeline=<span class="hljs-string">&quot;nateraw/vit-base-beans&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    data=data,
<span class="hljs-meta">&gt;&gt;&gt; </span>    metric=<span class="hljs-string">&quot;accuracy&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    input_column=<span class="hljs-string">&quot;image&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    label_column=<span class="hljs-string">&quot;labels&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    label_mapping={<span class="hljs-string">&#x27;angular_leaf_spot&#x27;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&#x27;bean_rust&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;healthy&#x27;</span>: <span class="hljs-number">2</span>},
<span class="hljs-meta">&gt;&gt;&gt; </span>    strategy=<span class="hljs-string">&quot;bootstrap&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    n_resamples=<span class="hljs-number">10</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    random_state=<span class="hljs-number">0</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>)`}}),{c(){u=s("p"),T=l("Examples:"),g=c(),y(p.$$.fragment)},l(o){u=n(o,"P",{});var h=r(u);T=i(h,"Examples:"),h.forEach(t),g=d(o),$(p.$$.fragment,o)},m(o,h){f(o,u,h),e(u,T),f(o,g,h),w(p,o,h),_=!0},p:vt,i(o){_||(E(p.$$.fragment,o),_=!0)},o(o){x(p.$$.fragment,o),_=!1},d(o){o&&t(u),o&&t(g),q(p,o)}}}function ao(O){let u,T,g,p,_,o,h,xe,la,He,pe,ia,We,z,R,qe,Y,ca,Te,da,Ge,ue,pa,Je,P,Z,ua,C,ma,me,fa,ga,je,ha,va,ke,_a,ba,ya,S,Ke,fe,$a,Qe,b,ee,wa,Pe,Ea,xa,B,ae,qa,A,Ta,Ce,ja,ka,De,Pa,Ca,Da,V,te,Na,Ne,Ia,Fa,H,oe,Ma,Ie,La,Ua,W,se,za,Fe,Aa,Oa,G,ne,Ra,Me,Sa,Xe,ge,Ba,Ye,D,re,Va,k,Ha,he,Wa,Ga,Le,Ja,Ka,Ue,Qa,Xa,ze,Ya,Za,et,M,le,at,Ae,tt,ot,J,Ze,N,ie,st,I,nt,ve,rt,lt,Oe,it,ct,Re,dt,pt,ut,L,ce,mt,Se,ft,gt,K,ea;return o=new Wt({}),Y=new Wt({}),Z=new F({props:{name:"evaluate.evaluator",anchor:"evaluate.evaluator",parameters:[{name:"task",val:": str = None"}],parametersDescription:[{anchor:"evaluate.evaluator.task",description:`<strong>task</strong> (<code>str</code>) &#x2014;
The task defining which evaluator will be returned. Currently accepted tasks are:<ul>
<li><code>&quot;text-classification&quot;</code> (alias <code>&quot;sentiment-analysis&quot;</code> available): will return a <a href="/docs/evaluate/pr_182/en/package_reference/evaluator_classes#evaluate.TextClassificationEvaluator">TextClassificationEvaluator</a>.</li>
<li><code>&quot;image-classification&quot;</code>: will return a <a href="/docs/evaluate/pr_182/en/package_reference/evaluator_classes#evaluate.ImageClassificationEvaluator">ImageClassificationEvaluator</a>.</li>
</ul>`,name:"task"}],source:"https://github.com/huggingface/evaluate/blob/vr_182/src/evaluate/evaluator/__init__.py#L73",returnDescription:`
<p>An evaluator suitable for the task.</p>
`,returnType:`
<p><a
  href="/docs/evaluate/pr_182/en/package_reference/evaluator_classes#evaluate.Evaluator"
>Evaluator</a></p>
`}}),S=new ht({props:{anchor:"evaluate.evaluator.example",$$slots:{default:[Yt]},$$scope:{ctx:O}}}),ee=new F({props:{name:"class evaluate.Evaluator",anchor:"evaluate.Evaluator",parameters:[{name:"task",val:": str"},{name:"default_metric_name",val:": str = None"}],source:"https://github.com/huggingface/evaluate/blob/vr_182/src/evaluate/evaluator/base.py#L54"}}),ae=new F({props:{name:"compute",anchor:"evaluate.Evaluator.compute",parameters:[{name:"model_or_pipeline",val:": typing.Union[str, ForwardRef('Pipeline'), typing.Callable, ForwardRef('PreTrainedModel'), ForwardRef('TFPreTrainedModel')] = None"},{name:"data",val:": typing.Union[str, datasets.arrow_dataset.Dataset] = None"},{name:"metric",val:": typing.Union[str, evaluate.module.EvaluationModule] = None"},{name:"tokenizer",val:": typing.Union[str, ForwardRef('PreTrainedTokenizer'), NoneType] = None"},{name:"strategy",val:": typing.Literal['simple', 'bootstrap'] = 'simple'"},{name:"confidence_level",val:": float = 0.95"},{name:"n_resamples",val:": int = 9999"},{name:"**compute_parameters",val:": typing.Dict"}],source:"https://github.com/huggingface/evaluate/blob/vr_182/src/evaluate/evaluator/base.py#L107"}}),te=new F({props:{name:"core_compute",anchor:"evaluate.Evaluator.core_compute",parameters:[{name:"references",val:": typing.List"},{name:"predictions",val:": typing.List"},{name:"metric",val:": EvaluationModule"},{name:"strategy",val:": typing.Literal['simple', 'bootstrap'] = 'simple'"},{name:"confidence_level",val:": float = 0.95"},{name:"n_resamples",val:": int = 9999"},{name:"random_state",val:": typing.Optional[int] = None"}],source:"https://github.com/huggingface/evaluate/blob/vr_182/src/evaluate/evaluator/base.py#L221"}}),oe=new F({props:{name:"prepare_data",anchor:"evaluate.Evaluator.prepare_data",parameters:[{name:"data",val:": typing.Union[str, datasets.arrow_dataset.Dataset]"},{name:"input_column",val:": str"},{name:"label_column",val:": str"}],parametersDescription:[{anchor:"evaluate.Evaluator.prepare_data.data",description:"<strong>data</strong> (<code>str</code> or <code>Dataset</code>, defaults to <code>None) -- Specifies the dataset we will run evaluation on. If it is of type </code>str`, we treat it as the dataset\nname, and load it. Otherwise we assume it represents a pre-loaded dataset.",name:"data"},{anchor:"evaluate.Evaluator.prepare_data.input_column",description:`<strong>input_column</strong> (<code>str</code>, defaults to <code>&quot;text&quot;</code>) &#x2014;
the name of the column containing the text feature in the dataset specified by <code>data</code>.`,name:"input_column"},{anchor:"evaluate.Evaluator.prepare_data.label_column",description:`<strong>label_column</strong> (<code>str</code>, defaults to <code>&quot;label&quot;</code>) &#x2014;
the name of the column containing the labels in the dataset specified by <code>data</code>.`,name:"label_column"}],source:"https://github.com/huggingface/evaluate/blob/vr_182/src/evaluate/evaluator/base.py#L125",returnDescription:`
<p>Loaded <code>datasets.Dataset</code>.</p>
`}}),se=new F({props:{name:"prepare_metric",anchor:"evaluate.Evaluator.prepare_metric",parameters:[{name:"metric",val:": typing.Union[str, evaluate.module.EvaluationModule]"}],parametersDescription:[{anchor:"evaluate.Evaluator.prepare_metric.metric",description:`<strong>metric</strong> (<code>str</code> or <code>EvaluationModule</code>, defaults to <code>None</code>) &#x2014;
Specifies the metric we use in evaluator. If it is of type <code>str</code>, we treat it as the metric name, and
load it. Otherwise we assume it represents a pre-loaded metric.`,name:"metric"}],source:"https://github.com/huggingface/evaluate/blob/vr_182/src/evaluate/evaluator/base.py#L197",returnDescription:`
<p>The loaded metric.</p>
`}}),ne=new F({props:{name:"prepare_pipeline",anchor:"evaluate.Evaluator.prepare_pipeline",parameters:[{name:"model_or_pipeline",val:": typing.Union[str, ForwardRef('Pipeline'), typing.Callable, ForwardRef('PreTrainedModel'), ForwardRef('TFPreTrainedModel')]"},{name:"preprocessor",val:": typing.Union[ForwardRef('PreTrainedTokenizerBase'), ForwardRef('FeatureExtractionMixin')] = None"}],parametersDescription:[{anchor:"evaluate.Evaluator.prepare_pipeline.model_or_pipeline",description:"<strong>model_or_pipeline</strong> (<code>str</code> or <code>Pipeline</code> or <code>Callable</code> or <code>PreTrainedModel</code> or <code>TFPreTrainedModel</code>, &#x2014;",name:"model_or_pipeline"},{anchor:"evaluate.Evaluator.prepare_pipeline.defaults",description:`<strong>defaults</strong> to <code>None</code>) &#x2014;
If the argument in not specified, we initialize the default pipeline for the task (in this case
<code>text-classification</code> or its alias - <code>sentiment-analysis</code>). If the argument is of the type <code>str</code> or
is a model instance, we use it to initialize a new <code>Pipeline</code> with the given model. Otherwise we assume the
argument specifies a pre-initialized pipeline.`,name:"defaults"},{anchor:"evaluate.Evaluator.prepare_pipeline.preprocessor",description:`<strong>preprocessor</strong> (<code>PreTrainedTokenizerBase</code> or <code>FeatureExtractionMixin</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Argument can be used to overwrite a default preprocessor if <code>model_or_pipeline</code> represents a model for
which we build a pipeline. If <code>model_or_pipeline</code> is <code>None</code> or a pre-initialized pipeline, we ignore
this argument.`,name:"preprocessor"}],source:"https://github.com/huggingface/evaluate/blob/vr_182/src/evaluate/evaluator/base.py#L156",returnDescription:`
<p>The initialized pipeline.</p>
`}}),re=new F({props:{name:"class evaluate.TextClassificationEvaluator",anchor:"evaluate.TextClassificationEvaluator",parameters:[{name:"task",val:" = 'text-classification'"},{name:"default_metric_name",val:" = None"}],source:"https://github.com/huggingface/evaluate/blob/vr_182/src/evaluate/evaluator/text_classification.py#L39"}}),le=new F({props:{name:"compute",anchor:"evaluate.TextClassificationEvaluator.compute",parameters:[{name:"model_or_pipeline",val:": typing.Union[str, ForwardRef('Pipeline'), typing.Callable, ForwardRef('PreTrainedModel'), ForwardRef('TFPreTrainedModel')] = None"},{name:"data",val:": typing.Union[str, datasets.arrow_dataset.Dataset] = None"},{name:"metric",val:": typing.Union[str, evaluate.module.EvaluationModule] = None"},{name:"tokenizer",val:": typing.Union[str, ForwardRef('PreTrainedTokenizer'), NoneType] = None"},{name:"strategy",val:": typing.Literal['simple', 'bootstrap'] = 'simple'"},{name:"confidence_level",val:": float = 0.95"},{name:"n_resamples",val:": int = 9999"},{name:"random_state",val:": typing.Optional[int] = None"},{name:"input_column",val:": str = 'text'"},{name:"label_column",val:": str = 'label'"},{name:"label_mapping",val:": typing.Union[typing.Dict[str, numbers.Number], NoneType] = None"}],parametersDescription:[{anchor:"evaluate.TextClassificationEvaluator.compute.model_or_pipeline",description:"<strong>model_or_pipeline</strong> (<code>str</code> or <code>Pipeline</code> or <code>Callable</code> or <code>PreTrainedModel</code> or <code>TFPreTrainedModel</code>, &#x2014;",name:"model_or_pipeline"},{anchor:"evaluate.TextClassificationEvaluator.compute.defaults",description:`<strong>defaults</strong> to <code>None</code>) &#x2014;
If the argument in not specified, we initialize the default pipeline for the task (in this case
<code>text-classification</code> or its alias - <code>sentiment-analysis</code>). If the argument is of the type <code>str</code> or
is a model instance, we use it to initialize a new <code>Pipeline</code> with the given model. Otherwise we assume the
argument specifies a pre-initialized pipeline.`,name:"defaults"},{anchor:"evaluate.TextClassificationEvaluator.compute.data",description:"<strong>data</strong> (<code>str</code> or <code>Dataset</code>, defaults to <code>None) -- Specifies the dataset we will run evaluation on. If it is of type </code>str`, we treat it as the dataset\nname, and load it. Otherwise we assume it represents a pre-loaded dataset.",name:"data"},{anchor:"evaluate.TextClassificationEvaluator.compute.metric",description:`<strong>metric</strong> (<code>str</code> or <code>EvaluationModule</code>, defaults to <code>None</code>) &#x2014;&#x201D;
Specifies the metric we use in evaluator. If it is of type <code>str</code>, we treat it as the metric name, and
load it. Otherwise we assume it represents a pre-loaded metric.
tokenizer &#x2014; (<code>str</code> or <code>PreTrainedTokenizer</code>, <em>optional</em>, defaults to <code>None</code>):
Argument can be used to overwrite a default tokenizer if <code>model_or_pipeline</code> represents a model for
which we build a pipeline. If <code>model_or_pipeline</code> is <code>None</code> or a pre-initialized pipeline, we ignore
this argument.
strategy &#x2014; (<code>Literal[&quot;simple&quot;, &quot;bootstrap&quot;]</code>, defaults to &#x201C;simple&#x201D;):
specifies the evaluation strategy. Possible values are:<ul>
<li><code>&quot;simple&quot;</code> - we evaluate the metric and return the scores.</li>
<li><code>&quot;bootstrap&quot;</code> - on top of computing the metric scores, we calculate the confidence interval for each
of the returned metric keys, using <code>scipy</code>&#x2019;s <code>bootstrap</code> method
<a href="https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html" rel="nofollow">https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html</a>.</li>
</ul>`,name:"metric"},{anchor:"evaluate.TextClassificationEvaluator.compute.confidence_level",description:`<strong>confidence_level</strong> (<code>float</code>, defaults to <code>0.95</code>) &#x2014;
The <code>confidence_level</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen.`,name:"confidence_level"},{anchor:"evaluate.TextClassificationEvaluator.compute.n_resamples",description:`<strong>n_resamples</strong> (<code>int</code>, defaults to <code>9999</code>) &#x2014;
The <code>n_resamples</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen.`,name:"n_resamples"},{anchor:"evaluate.TextClassificationEvaluator.compute.random_state",description:`<strong>random_state</strong> (<code>int</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
The <code>random_state</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen. Useful for
debugging.`,name:"random_state"},{anchor:"evaluate.TextClassificationEvaluator.compute.input_column",description:`<strong>input_column</strong> (<code>str</code>, defaults to <code>&quot;text&quot;</code>) &#x2014;
the name of the column containing the text feature in the dataset specified by <code>data</code>.`,name:"input_column"},{anchor:"evaluate.TextClassificationEvaluator.compute.label_column",description:`<strong>label_column</strong> (<code>str</code>, defaults to <code>&quot;label&quot;</code>) &#x2014;
the name of the column containing the labels in the dataset specified by <code>data</code>.`,name:"label_column"},{anchor:"evaluate.TextClassificationEvaluator.compute.label_mapping",description:`<strong>label_mapping</strong> (<code>Dict[str, Number]</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
We want to map class labels defined by the model in the pipeline to values consistent with those
defined in the <code>label_column</code> of the <code>data</code> dataset.`,name:"label_mapping"}],source:"https://github.com/huggingface/evaluate/blob/vr_182/src/evaluate/evaluator/text_classification.py#L51",returnDescription:`
<p>A <code>Dict</code>. The keys represent metric keys calculated for the <code>metric</code> spefied in function arguments. For the
<code>"simple"</code> strategy, the value is the metric score. For the <code>"bootstrap"</code> strategy, the value is a <code>Dict</code>
containing the score, the confidence interval and the standard error calculated for each metric key.</p>
`}}),J=new ht({props:{anchor:"evaluate.TextClassificationEvaluator.compute.example",$$slots:{default:[Zt]},$$scope:{ctx:O}}}),ie=new F({props:{name:"class evaluate.ImageClassificationEvaluator",anchor:"evaluate.ImageClassificationEvaluator",parameters:[{name:"task",val:" = 'image-classification'"},{name:"default_metric_name",val:" = None"}],source:"https://github.com/huggingface/evaluate/blob/vr_182/src/evaluate/evaluator/image_classification.py#L39"}}),ce=new F({props:{name:"compute",anchor:"evaluate.ImageClassificationEvaluator.compute",parameters:[{name:"model_or_pipeline",val:": typing.Union[str, ForwardRef('Pipeline'), typing.Callable, ForwardRef('PreTrainedModel'), ForwardRef('TFPreTrainedModel')] = None"},{name:"data",val:": typing.Union[str, datasets.arrow_dataset.Dataset] = None"},{name:"metric",val:": typing.Union[str, evaluate.module.EvaluationModule] = None"},{name:"feature_extractor",val:": typing.Union[str, ForwardRef('FeatureExtractionMixin'), NoneType] = None"},{name:"strategy",val:": typing.Literal['simple', 'bootstrap'] = 'simple'"},{name:"confidence_level",val:": float = 0.95"},{name:"n_resamples",val:": int = 9999"},{name:"random_state",val:": typing.Optional[int] = None"},{name:"input_column",val:": str = 'image'"},{name:"label_column",val:": str = 'labels'"},{name:"label_mapping",val:": typing.Union[typing.Dict[str, numbers.Number], NoneType] = None"}],parametersDescription:[{anchor:"evaluate.ImageClassificationEvaluator.compute.model_or_pipeline",description:"<strong>model_or_pipeline</strong> (<code>str</code> or <code>Pipeline</code> or <code>Callable</code> or <code>PreTrainedModel</code> or <code>TFPreTrainedModel</code>, &#x2014;",name:"model_or_pipeline"},{anchor:"evaluate.ImageClassificationEvaluator.compute.defaults",description:`<strong>defaults</strong> to <code>None</code>) &#x2014;
If the argument in not specified, we initialize the default pipeline for the task (in this case
<code>image-classification</code>. If the argument is of the type <code>str</code> or
is a model instance, we use it to initialize a new <code>Pipeline</code> with the given model. Otherwise we assume the
argument specifies a pre-initialized pipeline.`,name:"defaults"},{anchor:"evaluate.ImageClassificationEvaluator.compute.data",description:"<strong>data</strong> (<code>str</code> or <code>Dataset</code>, defaults to <code>None) -- Specifies the dataset we will run evaluation on. If it is of type </code>str`, we treat it as the dataset\nname, and load it. Otherwise we assume it represents a pre-loaded dataset.",name:"data"},{anchor:"evaluate.ImageClassificationEvaluator.compute.metric",description:`<strong>metric</strong> (<code>str</code> or <code>EvaluationModule</code>, defaults to <code>None</code>&#x201D; &#x2014;
Specifies the metric we use in evaluator. If it is of type <code>str</code>, we treat it as the metric name, and
load it. Otherwise we assume it represents a pre-loaded metric.
feature_extractor &#x2014; (<code>str</code> or <code>FeatureExtractionMixin</code>, <em>optional</em>, defaults to <code>None</code>):
Argument can be used to overwrite a default feature extractor if <code>model_or_pipeline</code> represents a model for
which we build a pipeline. If <code>model_or_pipeline</code> is <code>None</code> or a pre-initialized pipeline, we ignore
this argument.
strategy &#x2014; (<code>Literal[&quot;simple&quot;, &quot;bootstrap&quot;]</code>, defaults to &#x201C;simple&#x201D;):
specifies the evaluation strategy. Possible values are:<ul>
<li><code>&quot;simple&quot;</code> - we evaluate the metric and return the scores.</li>
<li><code>&quot;bootstrap&quot;</code> - on top of computing the metric scores, we calculate the confidence interval for each
of the returned metric keys, using <code>scipy</code>&#x2019;s <code>bootstrap</code> method
<a href="https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html" rel="nofollow">https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html</a>.</li>
</ul>`,name:"metric"},{anchor:"evaluate.ImageClassificationEvaluator.compute.confidence_level",description:`<strong>confidence_level</strong> (<code>float</code>, defaults to <code>0.95</code>) &#x2014;
The <code>confidence_level</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen.`,name:"confidence_level"},{anchor:"evaluate.ImageClassificationEvaluator.compute.n_resamples",description:`<strong>n_resamples</strong> (<code>int</code>, defaults to <code>9999</code>) &#x2014;
The <code>n_resamples</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen.`,name:"n_resamples"},{anchor:"evaluate.ImageClassificationEvaluator.compute.random_state",description:`<strong>random_state</strong> (<code>int</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
The <code>random_state</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen. Useful for
debugging.`,name:"random_state"},{anchor:"evaluate.ImageClassificationEvaluator.compute.input_column",description:`<strong>input_column</strong> (<code>str</code>, defaults to <code>&quot;image&quot;</code>) &#x2014;
the name of the column containing the images as PIL ImageFile in the dataset specified by <code>data</code>.`,name:"input_column"},{anchor:"evaluate.ImageClassificationEvaluator.compute.label_column",description:`<strong>label_column</strong> (<code>str</code>, defaults to <code>&quot;labels&quot;</code>) &#x2014;
the name of the column containing the labels in the dataset specified by <code>data</code>.`,name:"label_column"},{anchor:"evaluate.ImageClassificationEvaluator.compute.label_mapping",description:`<strong>label_mapping</strong> (<code>Dict[str, Number]</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
We want to map class labels defined by the model in the pipeline to values consistent with those
defined in the <code>label_column</code> of the <code>data</code> dataset.`,name:"label_mapping"}],source:"https://github.com/huggingface/evaluate/blob/vr_182/src/evaluate/evaluator/image_classification.py#L50",returnDescription:`
<p>A <code>Dict</code>. The keys represent metric keys calculated for the <code>metric</code> spefied in function arguments. For the
<code>"simple"</code> strategy, the value is the metric score. For the <code>"bootstrap"</code> strategy, the value is a <code>Dict</code>
containing the score, the confidence interval and the standard error calculated for each metric key.</p>
`}}),K=new ht({props:{anchor:"evaluate.ImageClassificationEvaluator.compute.example",$$slots:{default:[eo]},$$scope:{ctx:O}}}),{c(){u=s("meta"),T=c(),g=s("h1"),p=s("a"),_=s("span"),y(o.$$.fragment),h=c(),xe=s("span"),la=l("Evaluator"),He=c(),pe=s("p"),ia=l("The evaluator classes for automatic evaluation."),We=c(),z=s("h2"),R=s("a"),qe=s("span"),y(Y.$$.fragment),ca=c(),Te=s("span"),da=l("Evaluator classes"),Ge=c(),ue=s("p"),pa=l("The main entry point for using the evaluator:"),Je=c(),P=s("div"),y(Z.$$.fragment),ua=c(),C=s("p"),ma=l("Utility factory method to build an "),me=s("a"),fa=l("Evaluator"),ga=l(`.
Evaluators encapsulate a task and a default metric name. They leverage `),je=s("code"),ha=l("pipeline"),va=l(" functionalify from "),ke=s("code"),_a=l("transformers"),ba=l(`
to simplify the evaluation of multiple combinations of models, datasets and metrics for a given task.`),ya=c(),y(S.$$.fragment),Ke=c(),fe=s("p"),$a=l("The base class for all evaluator classes:"),Qe=c(),b=s("div"),y(ee.$$.fragment),wa=c(),Pe=s("p"),Ea=l(`The Evaluator class is the class from which all evaluators inherit. Refer to this class for methods shared across
different evaluators.
Base class implementing evaluator operations.`),xa=c(),B=s("div"),y(ae.$$.fragment),qa=c(),A=s("p"),Ta=l("A core method of the "),Ce=s("code"),ja=l("Evaluator"),ka=l(` class, computes the metric value for a pipeline and dataset compatible
with the task specified by the `),De=s("code"),Pa=l("Evaluator"),Ca=l("."),Da=c(),V=s("div"),y(te.$$.fragment),Na=c(),Ne=s("p"),Ia=l("Compute and return metrics."),Fa=c(),H=s("div"),y(oe.$$.fragment),Ma=c(),Ie=s("p"),La=l("Prepare data."),Ua=c(),W=s("div"),y(se.$$.fragment),za=c(),Fe=s("p"),Aa=l("Prepare metric."),Oa=c(),G=s("div"),y(ne.$$.fragment),Ra=c(),Me=s("p"),Sa=l("Prepare pipeline."),Xe=c(),ge=s("p"),Ba=l("The class for text classification evaluation:"),Ye=c(),D=s("div"),y(re.$$.fragment),Va=c(),k=s("p"),Ha=l(`Text classification evaluator.
This text classification evaluator can currently be loaded from `),he=s("a"),Wa=l("evaluator()"),Ga=l(` using the default task name
`),Le=s("code"),Ja=l("text-classification"),Ka=l(" or with a "),Ue=s("code"),Qa=l('"sentiment-analysis"'),Xa=l(` alias.
Methods in this class assume a data format compatible with the `),ze=s("code"),Ya=l("TextClassificationPipeline"),Za=l(` - a single textual
feature as input and a categorical label as output.`),et=c(),M=s("div"),y(le.$$.fragment),at=c(),Ae=s("p"),tt=l("Compute the metric for a given pipeline and dataset combination."),ot=c(),y(J.$$.fragment),Ze=c(),N=s("div"),y(ie.$$.fragment),st=c(),I=s("p"),nt=l(`Image classification evaluator.
This image classification evaluator can currently be loaded from `),ve=s("a"),rt=l("evaluator()"),lt=l(` using the default task name
`),Oe=s("code"),it=l("image-classification"),ct=l(`.
Methods in this class assume a data format compatible with the `),Re=s("code"),dt=l("ImageClassificationPipeline"),pt=l("."),ut=c(),L=s("div"),y(ce.$$.fragment),mt=c(),Se=s("p"),ft=l("Compute the metric for a given pipeline and dataset combination."),gt=c(),y(K.$$.fragment),this.h()},l(a){const m=Qt('[data-svelte="svelte-1phssyn"]',document.head);u=n(m,"META",{name:!0,content:!0}),m.forEach(t),T=d(a),g=n(a,"H1",{class:!0});var de=r(g);p=n(de,"A",{id:!0,class:!0,href:!0});var Be=r(p);_=n(Be,"SPAN",{});var Ve=r(_);$(o.$$.fragment,Ve),Ve.forEach(t),Be.forEach(t),h=d(de),xe=n(de,"SPAN",{});var bt=r(xe);la=i(bt,"Evaluator"),bt.forEach(t),de.forEach(t),He=d(a),pe=n(a,"P",{});var yt=r(pe);ia=i(yt,"The evaluator classes for automatic evaluation."),yt.forEach(t),We=d(a),z=n(a,"H2",{class:!0});var aa=r(z);R=n(aa,"A",{id:!0,class:!0,href:!0});var $t=r(R);qe=n($t,"SPAN",{});var wt=r(qe);$(Y.$$.fragment,wt),wt.forEach(t),$t.forEach(t),ca=d(aa),Te=n(aa,"SPAN",{});var Et=r(Te);da=i(Et,"Evaluator classes"),Et.forEach(t),aa.forEach(t),Ge=d(a),ue=n(a,"P",{});var xt=r(ue);pa=i(xt,"The main entry point for using the evaluator:"),xt.forEach(t),Je=d(a),P=n(a,"DIV",{class:!0});var _e=r(P);$(Z.$$.fragment,_e),ua=d(_e),C=n(_e,"P",{});var Q=r(C);ma=i(Q,"Utility factory method to build an "),me=n(Q,"A",{href:!0});var qt=r(me);fa=i(qt,"Evaluator"),qt.forEach(t),ga=i(Q,`.
Evaluators encapsulate a task and a default metric name. They leverage `),je=n(Q,"CODE",{});var Tt=r(je);ha=i(Tt,"pipeline"),Tt.forEach(t),va=i(Q," functionalify from "),ke=n(Q,"CODE",{});var jt=r(ke);_a=i(jt,"transformers"),jt.forEach(t),ba=i(Q,`
to simplify the evaluation of multiple combinations of models, datasets and metrics for a given task.`),Q.forEach(t),ya=d(_e),$(S.$$.fragment,_e),_e.forEach(t),Ke=d(a),fe=n(a,"P",{});var kt=r(fe);$a=i(kt,"The base class for all evaluator classes:"),kt.forEach(t),Qe=d(a),b=n(a,"DIV",{class:!0});var j=r(b);$(ee.$$.fragment,j),wa=d(j),Pe=n(j,"P",{});var Pt=r(Pe);Ea=i(Pt,`The Evaluator class is the class from which all evaluators inherit. Refer to this class for methods shared across
different evaluators.
Base class implementing evaluator operations.`),Pt.forEach(t),xa=d(j),B=n(j,"DIV",{class:!0});var ta=r(B);$(ae.$$.fragment,ta),qa=d(ta),A=n(ta,"P",{});var be=r(A);Ta=i(be,"A core method of the "),Ce=n(be,"CODE",{});var Ct=r(Ce);ja=i(Ct,"Evaluator"),Ct.forEach(t),ka=i(be,` class, computes the metric value for a pipeline and dataset compatible
with the task specified by the `),De=n(be,"CODE",{});var Dt=r(De);Pa=i(Dt,"Evaluator"),Dt.forEach(t),Ca=i(be,"."),be.forEach(t),ta.forEach(t),Da=d(j),V=n(j,"DIV",{class:!0});var oa=r(V);$(te.$$.fragment,oa),Na=d(oa),Ne=n(oa,"P",{});var Nt=r(Ne);Ia=i(Nt,"Compute and return metrics."),Nt.forEach(t),oa.forEach(t),Fa=d(j),H=n(j,"DIV",{class:!0});var sa=r(H);$(oe.$$.fragment,sa),Ma=d(sa),Ie=n(sa,"P",{});var It=r(Ie);La=i(It,"Prepare data."),It.forEach(t),sa.forEach(t),Ua=d(j),W=n(j,"DIV",{class:!0});var na=r(W);$(se.$$.fragment,na),za=d(na),Fe=n(na,"P",{});var Ft=r(Fe);Aa=i(Ft,"Prepare metric."),Ft.forEach(t),na.forEach(t),Oa=d(j),G=n(j,"DIV",{class:!0});var ra=r(G);$(ne.$$.fragment,ra),Ra=d(ra),Me=n(ra,"P",{});var Mt=r(Me);Sa=i(Mt,"Prepare pipeline."),Mt.forEach(t),ra.forEach(t),j.forEach(t),Xe=d(a),ge=n(a,"P",{});var Lt=r(ge);Ba=i(Lt,"The class for text classification evaluation:"),Lt.forEach(t),Ye=d(a),D=n(a,"DIV",{class:!0});var ye=r(D);$(re.$$.fragment,ye),Va=d(ye),k=n(ye,"P",{});var U=r(k);Ha=i(U,`Text classification evaluator.
This text classification evaluator can currently be loaded from `),he=n(U,"A",{href:!0});var Ut=r(he);Wa=i(Ut,"evaluator()"),Ut.forEach(t),Ga=i(U,` using the default task name
`),Le=n(U,"CODE",{});var zt=r(Le);Ja=i(zt,"text-classification"),zt.forEach(t),Ka=i(U," or with a "),Ue=n(U,"CODE",{});var At=r(Ue);Qa=i(At,'"sentiment-analysis"'),At.forEach(t),Xa=i(U,` alias.
Methods in this class assume a data format compatible with the `),ze=n(U,"CODE",{});var Ot=r(ze);Ya=i(Ot,"TextClassificationPipeline"),Ot.forEach(t),Za=i(U,` - a single textual
feature as input and a categorical label as output.`),U.forEach(t),et=d(ye),M=n(ye,"DIV",{class:!0});var $e=r(M);$(le.$$.fragment,$e),at=d($e),Ae=n($e,"P",{});var Rt=r(Ae);tt=i(Rt,"Compute the metric for a given pipeline and dataset combination."),Rt.forEach(t),ot=d($e),$(J.$$.fragment,$e),$e.forEach(t),ye.forEach(t),Ze=d(a),N=n(a,"DIV",{class:!0});var we=r(N);$(ie.$$.fragment,we),st=d(we),I=n(we,"P",{});var X=r(I);nt=i(X,`Image classification evaluator.
This image classification evaluator can currently be loaded from `),ve=n(X,"A",{href:!0});var St=r(ve);rt=i(St,"evaluator()"),St.forEach(t),lt=i(X,` using the default task name
`),Oe=n(X,"CODE",{});var Bt=r(Oe);it=i(Bt,"image-classification"),Bt.forEach(t),ct=i(X,`.
Methods in this class assume a data format compatible with the `),Re=n(X,"CODE",{});var Vt=r(Re);dt=i(Vt,"ImageClassificationPipeline"),Vt.forEach(t),pt=i(X,"."),X.forEach(t),ut=d(we),L=n(we,"DIV",{class:!0});var Ee=r(L);$(ce.$$.fragment,Ee),mt=d(Ee),Se=n(Ee,"P",{});var Ht=r(Se);ft=i(Ht,"Compute the metric for a given pipeline and dataset combination."),Ht.forEach(t),gt=d(Ee),$(K.$$.fragment,Ee),Ee.forEach(t),we.forEach(t),this.h()},h(){v(u,"name","hf:doc:metadata"),v(u,"content",JSON.stringify(to)),v(p,"id","evaluator"),v(p,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),v(p,"href","#evaluator"),v(g,"class","relative group"),v(R,"id","evaluate.evaluator"),v(R,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),v(R,"href","#evaluate.evaluator"),v(z,"class","relative group"),v(me,"href","/docs/evaluate/pr_182/en/package_reference/evaluator_classes#evaluate.Evaluator"),v(P,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(B,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(V,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(H,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(W,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(G,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(b,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(he,"href","/docs/evaluate/pr_182/en/package_reference/evaluator_classes#evaluate.evaluator"),v(M,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(D,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(ve,"href","/docs/evaluate/pr_182/en/package_reference/evaluator_classes#evaluate.evaluator"),v(L,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),v(N,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(a,m){e(document.head,u),f(a,T,m),f(a,g,m),e(g,p),e(p,_),w(o,_,null),e(g,h),e(g,xe),e(xe,la),f(a,He,m),f(a,pe,m),e(pe,ia),f(a,We,m),f(a,z,m),e(z,R),e(R,qe),w(Y,qe,null),e(z,ca),e(z,Te),e(Te,da),f(a,Ge,m),f(a,ue,m),e(ue,pa),f(a,Je,m),f(a,P,m),w(Z,P,null),e(P,ua),e(P,C),e(C,ma),e(C,me),e(me,fa),e(C,ga),e(C,je),e(je,ha),e(C,va),e(C,ke),e(ke,_a),e(C,ba),e(P,ya),w(S,P,null),f(a,Ke,m),f(a,fe,m),e(fe,$a),f(a,Qe,m),f(a,b,m),w(ee,b,null),e(b,wa),e(b,Pe),e(Pe,Ea),e(b,xa),e(b,B),w(ae,B,null),e(B,qa),e(B,A),e(A,Ta),e(A,Ce),e(Ce,ja),e(A,ka),e(A,De),e(De,Pa),e(A,Ca),e(b,Da),e(b,V),w(te,V,null),e(V,Na),e(V,Ne),e(Ne,Ia),e(b,Fa),e(b,H),w(oe,H,null),e(H,Ma),e(H,Ie),e(Ie,La),e(b,Ua),e(b,W),w(se,W,null),e(W,za),e(W,Fe),e(Fe,Aa),e(b,Oa),e(b,G),w(ne,G,null),e(G,Ra),e(G,Me),e(Me,Sa),f(a,Xe,m),f(a,ge,m),e(ge,Ba),f(a,Ye,m),f(a,D,m),w(re,D,null),e(D,Va),e(D,k),e(k,Ha),e(k,he),e(he,Wa),e(k,Ga),e(k,Le),e(Le,Ja),e(k,Ka),e(k,Ue),e(Ue,Qa),e(k,Xa),e(k,ze),e(ze,Ya),e(k,Za),e(D,et),e(D,M),w(le,M,null),e(M,at),e(M,Ae),e(Ae,tt),e(M,ot),w(J,M,null),f(a,Ze,m),f(a,N,m),w(ie,N,null),e(N,st),e(N,I),e(I,nt),e(I,ve),e(ve,rt),e(I,lt),e(I,Oe),e(Oe,it),e(I,ct),e(I,Re),e(Re,dt),e(I,pt),e(N,ut),e(N,L),w(ce,L,null),e(L,mt),e(L,Se),e(Se,ft),e(L,gt),w(K,L,null),ea=!0},p(a,[m]){const de={};m&2&&(de.$$scope={dirty:m,ctx:a}),S.$set(de);const Be={};m&2&&(Be.$$scope={dirty:m,ctx:a}),J.$set(Be);const Ve={};m&2&&(Ve.$$scope={dirty:m,ctx:a}),K.$set(Ve)},i(a){ea||(E(o.$$.fragment,a),E(Y.$$.fragment,a),E(Z.$$.fragment,a),E(S.$$.fragment,a),E(ee.$$.fragment,a),E(ae.$$.fragment,a),E(te.$$.fragment,a),E(oe.$$.fragment,a),E(se.$$.fragment,a),E(ne.$$.fragment,a),E(re.$$.fragment,a),E(le.$$.fragment,a),E(J.$$.fragment,a),E(ie.$$.fragment,a),E(ce.$$.fragment,a),E(K.$$.fragment,a),ea=!0)},o(a){x(o.$$.fragment,a),x(Y.$$.fragment,a),x(Z.$$.fragment,a),x(S.$$.fragment,a),x(ee.$$.fragment,a),x(ae.$$.fragment,a),x(te.$$.fragment,a),x(oe.$$.fragment,a),x(se.$$.fragment,a),x(ne.$$.fragment,a),x(re.$$.fragment,a),x(le.$$.fragment,a),x(J.$$.fragment,a),x(ie.$$.fragment,a),x(ce.$$.fragment,a),x(K.$$.fragment,a),ea=!1},d(a){t(u),a&&t(T),a&&t(g),q(o),a&&t(He),a&&t(pe),a&&t(We),a&&t(z),q(Y),a&&t(Ge),a&&t(ue),a&&t(Je),a&&t(P),q(Z),q(S),a&&t(Ke),a&&t(fe),a&&t(Qe),a&&t(b),q(ee),q(ae),q(te),q(oe),q(se),q(ne),a&&t(Xe),a&&t(ge),a&&t(Ye),a&&t(D),q(re),q(le),q(J),a&&t(Ze),a&&t(N),q(ie),q(ce),q(K)}}}const to={local:"evaluator",sections:[{local:"evaluate.evaluator",title:"Evaluator classes"}],title:"Evaluator"};function oo(O){return Xt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class co extends Gt{constructor(u){super();Jt(this,u,oo,ao,Kt,{})}}export{co as default,to as metadata};
