import{S as aa,i as ea,s as na,e as l,k as d,w as ss,t as n,M as ta,c as o,d as e,m as g,a as p,x as as,h as t,b as q,G as a,g as i,y as es,L as la,q as ns,o as ts,B as ls,v as oa}from"../chunks/vendor-hf-doc-builder.js";import{I as ca}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as Bs}from"../chunks/CodeBlock-hf-doc-builder.js";function pa(Fs){let f,G,j,x,A,v,os,D,cs,J,h,ps,O,rs,is,k,us,_s,R,u,hs,L,ms,ds,T,gs,fs,P,js,xs,U,r,bs,H,vs,ys,B,Es,ws,F,qs,ks,M,$s,Ss,N,Cs,As,z,y,Q,m,Ds,I,Os,Ls,K,Ts,Ps,V,E,W,$,Hs,X,w,Y;return v=new ca({}),y=new Bs({props:{code:`import evaluate
from evaluate.evaluation_suite import SubTask

class Suite(evaluate.EvaluationSuite):

    def __init__(self, name):
        super().__init__(name)
        self.preprocessor = lambda x: {"text": x["text"].lower()}
        self.suite = [
            SubTask(
                task_type="text-classification",
                data="glue",
                subset="cola",
                split="test[:10]",
                args_for_task={
                    "metric": "accuracy",
                    "input_column": "sentence",
                    "label_column": "label",
                    "label_mapping": {
                        "LABEL_0": 0.0,
                        "LABEL_1": 1.0
                    }
                }
            )
        ]`,highlighted:`<span class="hljs-keyword">import</span> evaluate
<span class="hljs-keyword">from</span> evaluate.evaluation_suite <span class="hljs-keyword">import</span> SubTask

<span class="hljs-keyword">class</span> <span class="hljs-title class_">Suite</span>(evaluate.EvaluationSuite):

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, name</span>):
        <span class="hljs-built_in">super</span>().__init__(name)
        self.preprocessor = <span class="hljs-keyword">lambda</span> x: {<span class="hljs-string">&quot;text&quot;</span>: x[<span class="hljs-string">&quot;text&quot;</span>].lower()}
        self.suite = [
            SubTask(
                task_type=<span class="hljs-string">&quot;text-classification&quot;</span>,
                data=<span class="hljs-string">&quot;glue&quot;</span>,
                subset=<span class="hljs-string">&quot;cola&quot;</span>,
                split=<span class="hljs-string">&quot;test[:10]&quot;</span>,
                args_for_task={
                    <span class="hljs-string">&quot;metric&quot;</span>: <span class="hljs-string">&quot;accuracy&quot;</span>,
                    <span class="hljs-string">&quot;input_column&quot;</span>: <span class="hljs-string">&quot;sentence&quot;</span>,
                    <span class="hljs-string">&quot;label_column&quot;</span>: <span class="hljs-string">&quot;label&quot;</span>,
                    <span class="hljs-string">&quot;label_mapping&quot;</span>: {
                        <span class="hljs-string">&quot;LABEL_0&quot;</span>: <span class="hljs-number">0.0</span>,
                        <span class="hljs-string">&quot;LABEL_1&quot;</span>: <span class="hljs-number">1.0</span>
                    }
                }
            )
        ]`}}),E=new Bs({props:{code:`from evaluate import EvaluationSuite

suite = EvaluationSuite.load('mathemakitten/glue')
results = suite.run("gpt2")`,highlighted:`<span class="hljs-keyword">from</span> evaluate <span class="hljs-keyword">import</span> EvaluationSuite

suite = EvaluationSuite.load(<span class="hljs-string">&#x27;mathemakitten/glue&#x27;</span>)
results = suite.run(<span class="hljs-string">&quot;gpt2&quot;</span>)`}}),w=new Bs({props:{code:"{'glue/cola': {'accuracy': 0.0, 'total_time_in_seconds': 0.9766696180449799, 'samples_per_second': 10.238876909079256, 'latency_in_seconds': 0.09766696180449798}, 'glue/sst2': {'accuracy': 0.5, 'total_time_in_seconds': 1.1422595420153812, 'samples_per_second': 8.754577775166744, 'latency_in_seconds': 0.11422595420153811}, 'glue/qqp': {'accuracy': 0.6, 'total_time_in_seconds': 1.3553926559980027, 'samples_per_second': 7.377935800188323, 'latency_in_seconds': 0.13553926559980026}, 'glue/mrpc': {'accuracy': 0.6, 'total_time_in_seconds': 2.021696529001929, 'samples_per_second': 4.946340786832532, 'latency_in_seconds': 0.2021696529001929}, 'glue/mnli': {'accuracy': 0.2, 'total_time_in_seconds': 2.0380110969999805, 'samples_per_second': 4.9067446270142145, 'latency_in_seconds': 0.20380110969999807}, 'glue/qnli': {'accuracy': 0.3, 'total_time_in_seconds': 2.082032073987648, 'samples_per_second': 4.802999975330509, 'latency_in_seconds': 0.20820320739876477}, 'glue/rte': {'accuracy': 0.7, 'total_time_in_seconds': 2.8592985830036923, 'samples_per_second': 3.4973612267855576, 'latency_in_seconds': 0.2859298583003692}, 'glue/wnli': {'accuracy': 0.5, 'total_time_in_seconds': 1.5406486629508436, 'samples_per_second': 6.490772517107661, 'latency_in_seconds': 0.15406486629508437}}",highlighted:'{<span class="hljs-string">&#x27;glue/cola&#x27;</span>: {<span class="hljs-string">&#x27;accuracy&#x27;</span>: <span class="hljs-number">0.0</span>, <span class="hljs-string">&#x27;total_time_in_seconds&#x27;</span>: <span class="hljs-number">0.9766696180449799</span>, <span class="hljs-string">&#x27;samples_per_second&#x27;</span>: <span class="hljs-number">10.238876909079256</span>, <span class="hljs-string">&#x27;latency_in_seconds&#x27;</span>: <span class="hljs-number">0.09766696180449798</span>}, <span class="hljs-string">&#x27;glue/sst2&#x27;</span>: {<span class="hljs-string">&#x27;accuracy&#x27;</span>: <span class="hljs-number">0.5</span>, <span class="hljs-string">&#x27;total_time_in_seconds&#x27;</span>: <span class="hljs-number">1.1422595420153812</span>, <span class="hljs-string">&#x27;samples_per_second&#x27;</span>: <span class="hljs-number">8.754577775166744</span>, <span class="hljs-string">&#x27;latency_in_seconds&#x27;</span>: <span class="hljs-number">0.11422595420153811</span>}, <span class="hljs-string">&#x27;glue/qqp&#x27;</span>: {<span class="hljs-string">&#x27;accuracy&#x27;</span>: <span class="hljs-number">0.6</span>, <span class="hljs-string">&#x27;total_time_in_seconds&#x27;</span>: <span class="hljs-number">1.3553926559980027</span>, <span class="hljs-string">&#x27;samples_per_second&#x27;</span>: <span class="hljs-number">7.377935800188323</span>, <span class="hljs-string">&#x27;latency_in_seconds&#x27;</span>: <span class="hljs-number">0.13553926559980026</span>}, <span class="hljs-string">&#x27;glue/mrpc&#x27;</span>: {<span class="hljs-string">&#x27;accuracy&#x27;</span>: <span class="hljs-number">0.6</span>, <span class="hljs-string">&#x27;total_time_in_seconds&#x27;</span>: <span class="hljs-number">2.021696529001929</span>, <span class="hljs-string">&#x27;samples_per_second&#x27;</span>: <span class="hljs-number">4.946340786832532</span>, <span class="hljs-string">&#x27;latency_in_seconds&#x27;</span>: <span class="hljs-number">0.2021696529001929</span>}, <span class="hljs-string">&#x27;glue/mnli&#x27;</span>: {<span class="hljs-string">&#x27;accuracy&#x27;</span>: <span class="hljs-number">0.2</span>, <span class="hljs-string">&#x27;total_time_in_seconds&#x27;</span>: <span class="hljs-number">2.0380110969999805</span>, <span class="hljs-string">&#x27;samples_per_second&#x27;</span>: <span class="hljs-number">4.9067446270142145</span>, <span class="hljs-string">&#x27;latency_in_seconds&#x27;</span>: <span class="hljs-number">0.20380110969999807</span>}, <span class="hljs-string">&#x27;glue/qnli&#x27;</span>: {<span class="hljs-string">&#x27;accuracy&#x27;</span>: <span class="hljs-number">0.3</span>, <span class="hljs-string">&#x27;total_time_in_seconds&#x27;</span>: <span class="hljs-number">2.082032073987648</span>, <span class="hljs-string">&#x27;samples_per_second&#x27;</span>: <span class="hljs-number">4.802999975330509</span>, <span class="hljs-string">&#x27;latency_in_seconds&#x27;</span>: <span class="hljs-number">0.20820320739876477</span>}, <span class="hljs-string">&#x27;glue/rte&#x27;</span>: {<span class="hljs-string">&#x27;accuracy&#x27;</span>: <span class="hljs-number">0.7</span>, <span class="hljs-string">&#x27;total_time_in_seconds&#x27;</span>: <span class="hljs-number">2.8592985830036923</span>, <span class="hljs-string">&#x27;samples_per_second&#x27;</span>: <span class="hljs-number">3.4973612267855576</span>, <span class="hljs-string">&#x27;latency_in_seconds&#x27;</span>: <span class="hljs-number">0.2859298583003692</span>}, <span class="hljs-string">&#x27;glue/wnli&#x27;</span>: {<span class="hljs-string">&#x27;accuracy&#x27;</span>: <span class="hljs-number">0.5</span>, <span class="hljs-string">&#x27;total_time_in_seconds&#x27;</span>: <span class="hljs-number">1.5406486629508436</span>, <span class="hljs-string">&#x27;samples_per_second&#x27;</span>: <span class="hljs-number">6.490772517107661</span>, <span class="hljs-string">&#x27;latency_in_seconds&#x27;</span>: <span class="hljs-number">0.15406486629508437</span>}}'}}),{c(){f=l("meta"),G=d(),j=l("h1"),x=l("a"),A=l("span"),ss(v.$$.fragment),os=d(),D=l("span"),cs=n("Creating an EvaluationSuite"),J=d(),h=l("p"),ps=n("The "),O=l("code"),rs=n("EvaluationSuite"),is=n(" provides a way to compose any number of ("),k=l("a"),us=n("evaluator"),_s=n(", dataset, metric) tuples to evaluate a model on a collection of several evaluation tasks."),R=d(),u=l("p"),hs=n("A new "),L=l("code"),ms=n("EvaluationSuite"),ds=n(" is made up of "),T=l("code"),gs=n("SubTask"),fs=n(" classes, and can be defined by subclassing the "),P=l("code"),js=n("EvaluationSuite"),xs=n(" class. Files can be uploaded to a Space on the Hugging Face Hub or saved locally."),U=d(),r=l("p"),bs=n("Datasets which require additional preprocessing before being used with an "),H=l("code"),vs=n("Evaluator"),ys=n(" can be processed with "),B=l("code"),Es=n("datasets"),ws=n(" transformations by setting the "),F=l("code"),qs=n("preprocessor"),ks=n(" attribute to a preprocessing function. Keyword arguments for the "),M=l("code"),$s=n("Evaluator"),Ss=n(" can be passed down through "),N=l("code"),Cs=n("args_for_task"),As=n("."),z=d(),ss(y.$$.fragment),Q=d(),m=l("p"),Ds=n("An "),I=l("code"),Os=n("EvaluationSuite"),Ls=n(" can be loaded by name from the Hugging Face Hub, or locally by providing a path, and run with "),K=l("code"),Ts=n(".run(model_or_pipeline)"),Ps=n("."),V=d(),ss(E.$$.fragment),W=d(),$=l("p"),Hs=n("The evaluation results are printed along with their task names and information about the time it took to obtain predictions through the pipeline."),X=d(),ss(w.$$.fragment),this.h()},l(s){const c=ta('[data-svelte="svelte-1phssyn"]',document.head);f=o(c,"META",{name:!0,content:!0}),c.forEach(e),G=g(s),j=o(s,"H1",{class:!0});var Z=p(j);x=o(Z,"A",{id:!0,class:!0,href:!0});var Ms=p(x);A=o(Ms,"SPAN",{});var Ns=p(A);as(v.$$.fragment,Ns),Ns.forEach(e),Ms.forEach(e),os=g(Z),D=o(Z,"SPAN",{});var Is=p(D);cs=t(Is,"Creating an EvaluationSuite"),Is.forEach(e),Z.forEach(e),J=g(s),h=o(s,"P",{});var S=p(h);ps=t(S,"The "),O=o(S,"CODE",{});var Ks=p(O);rs=t(Ks,"EvaluationSuite"),Ks.forEach(e),is=t(S," provides a way to compose any number of ("),k=o(S,"A",{href:!0});var Gs=p(k);us=t(Gs,"evaluator"),Gs.forEach(e),_s=t(S,", dataset, metric) tuples to evaluate a model on a collection of several evaluation tasks."),S.forEach(e),R=g(s),u=o(s,"P",{});var b=p(u);hs=t(b,"A new "),L=o(b,"CODE",{});var Js=p(L);ms=t(Js,"EvaluationSuite"),Js.forEach(e),ds=t(b," is made up of "),T=o(b,"CODE",{});var Rs=p(T);gs=t(Rs,"SubTask"),Rs.forEach(e),fs=t(b," classes, and can be defined by subclassing the "),P=o(b,"CODE",{});var Us=p(P);js=t(Us,"EvaluationSuite"),Us.forEach(e),xs=t(b," class. Files can be uploaded to a Space on the Hugging Face Hub or saved locally."),b.forEach(e),U=g(s),r=o(s,"P",{});var _=p(r);bs=t(_,"Datasets which require additional preprocessing before being used with an "),H=o(_,"CODE",{});var zs=p(H);vs=t(zs,"Evaluator"),zs.forEach(e),ys=t(_," can be processed with "),B=o(_,"CODE",{});var Qs=p(B);Es=t(Qs,"datasets"),Qs.forEach(e),ws=t(_," transformations by setting the "),F=o(_,"CODE",{});var Vs=p(F);qs=t(Vs,"preprocessor"),Vs.forEach(e),ks=t(_," attribute to a preprocessing function. Keyword arguments for the "),M=o(_,"CODE",{});var Ws=p(M);$s=t(Ws,"Evaluator"),Ws.forEach(e),Ss=t(_," can be passed down through "),N=o(_,"CODE",{});var Xs=p(N);Cs=t(Xs,"args_for_task"),Xs.forEach(e),As=t(_,"."),_.forEach(e),z=g(s),as(y.$$.fragment,s),Q=g(s),m=o(s,"P",{});var C=p(m);Ds=t(C,"An "),I=o(C,"CODE",{});var Ys=p(I);Os=t(Ys,"EvaluationSuite"),Ys.forEach(e),Ls=t(C," can be loaded by name from the Hugging Face Hub, or locally by providing a path, and run with "),K=o(C,"CODE",{});var Zs=p(K);Ts=t(Zs,".run(model_or_pipeline)"),Zs.forEach(e),Ps=t(C,"."),C.forEach(e),V=g(s),as(E.$$.fragment,s),W=g(s),$=o(s,"P",{});var sa=p($);Hs=t(sa,"The evaluation results are printed along with their task names and information about the time it took to obtain predictions through the pipeline."),sa.forEach(e),X=g(s),as(w.$$.fragment,s),this.h()},h(){q(f,"name","hf:doc:metadata"),q(f,"content",JSON.stringify(ra)),q(x,"id","creating-an-evaluationsuite"),q(x,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),q(x,"href","#creating-an-evaluationsuite"),q(j,"class","relative group"),q(k,"href","base_evaluator")},m(s,c){a(document.head,f),i(s,G,c),i(s,j,c),a(j,x),a(x,A),es(v,A,null),a(j,os),a(j,D),a(D,cs),i(s,J,c),i(s,h,c),a(h,ps),a(h,O),a(O,rs),a(h,is),a(h,k),a(k,us),a(h,_s),i(s,R,c),i(s,u,c),a(u,hs),a(u,L),a(L,ms),a(u,ds),a(u,T),a(T,gs),a(u,fs),a(u,P),a(P,js),a(u,xs),i(s,U,c),i(s,r,c),a(r,bs),a(r,H),a(H,vs),a(r,ys),a(r,B),a(B,Es),a(r,ws),a(r,F),a(F,qs),a(r,ks),a(r,M),a(M,$s),a(r,Ss),a(r,N),a(N,Cs),a(r,As),i(s,z,c),es(y,s,c),i(s,Q,c),i(s,m,c),a(m,Ds),a(m,I),a(I,Os),a(m,Ls),a(m,K),a(K,Ts),a(m,Ps),i(s,V,c),es(E,s,c),i(s,W,c),i(s,$,c),a($,Hs),i(s,X,c),es(w,s,c),Y=!0},p:la,i(s){Y||(ns(v.$$.fragment,s),ns(y.$$.fragment,s),ns(E.$$.fragment,s),ns(w.$$.fragment,s),Y=!0)},o(s){ts(v.$$.fragment,s),ts(y.$$.fragment,s),ts(E.$$.fragment,s),ts(w.$$.fragment,s),Y=!1},d(s){e(f),s&&e(G),s&&e(j),ls(v),s&&e(J),s&&e(h),s&&e(R),s&&e(u),s&&e(U),s&&e(r),s&&e(z),ls(y,s),s&&e(Q),s&&e(m),s&&e(V),ls(E,s),s&&e(W),s&&e($),s&&e(X),ls(w,s)}}}const ra={local:"creating-an-evaluationsuite",title:"Creating an EvaluationSuite"};function ia(Fs){return oa(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ma extends aa{constructor(f){super();ea(this,f,ia,pa,na,{})}}export{ma as default,ra as metadata};
