import{S as Ta,i as Ma,s as Ia,e as r,k as c,w as Re,t as s,M as Sa,c as o,d as t,m as p,a as n,x as Fe,h as i,b as l,G as a,g as f,y as Ue,L as Ca,q as ze,o as Be,B as De,v as Na}from"../chunks/vendor-hf-doc-builder.js";import{I as We}from"../chunks/IconCopyLink-hf-doc-builder.js";function La(jt){let y,ge,_,x,ie,I,Je,le,je,we,j,Ke,ye,K,Qe,_e,E,A,he,S,Ve,fe,Xe,Ee,Q,Ye,be,d,V,C,Ze,et,tt,X,N,at,rt,ot,Y,L,st,nt,$e,v,it,H,lt,ht,q,ft,ct,xe,b,k,ce,G,pt,pe,ut,Ae,Z,mt,ke,u,dt,O,vt,gt,ue,wt,yt,me,_t,Et,Pe,ee,bt,Te,$,P,de,R,$t,ve,xt,Me,te,At,Ie,g,kt,F,Pt,Tt,U,Mt,It,Se,m,St,z,Ct,Nt,B,Lt,Ht,D,qt,Gt,Ce,ae,Ot,Ne,w,Rt,W,Ft,Ut,J,zt,Bt,Le;return I=new We({}),S=new We({}),G=new We({}),R=new We({}),{c(){y=r("meta"),ge=c(),_=r("h1"),x=r("a"),ie=r("span"),Re(I.$$.fragment),Je=c(),le=r("span"),je=s("Types of Evaluations in \u{1F917} Evaluate"),we=c(),j=r("p"),Ke=s("The goal of the \u{1F917} Evaluate library is to support different types of evaluation, depending on different goals, datasets and models."),ye=c(),K=r("p"),Qe=s("Here are the types of evaluations that are currently supported with a few examples for each:"),_e=c(),E=r("h2"),A=r("a"),he=r("span"),Re(S.$$.fragment),Ve=c(),fe=r("span"),Xe=s("Metrics"),Ee=s(`

A metric measures the performance of a model on a given dataset. This is often based on an existing ground truth (i.e. a set of references), but there are also *referenceless metrics* which allow evaluating generated text by leveraging a pretrained model such as [GPT-2](https://huggingface.co/gpt2).
`),Q=r("p"),Ye=s("Examples of metrics include:"),be=c(),d=r("ul"),V=r("li"),C=r("a"),Ze=s("Accuracy"),et=s(" : the proportion of correct predictions among the total number of cases processed."),tt=c(),X=r("li"),N=r("a"),at=s("Exact Match"),rt=s(": the rate at which the input predicted strings exactly match their references."),ot=c(),Y=r("li"),L=r("a"),st=s("Mean Intersection over union (IoUO)"),nt=s(": the area of overlap between the predicted segmentation of an image and the ground truth divided by the area of union between the predicted segmentation and the ground truth."),$e=c(),v=r("p"),it=s("Metrics are often used to track model performance on benchmark datasets, and to report progress on tasks such as "),H=r("a"),lt=s("machine translation"),ht=s(" and "),q=r("a"),ft=s("image classification"),ct=s("."),xe=c(),b=r("h2"),k=r("a"),ce=r("span"),Re(G.$$.fragment),pt=c(),pe=r("span"),ut=s("Comparisons"),Ae=c(),Z=r("p"),mt=s("Comparisons can be useful to compare the performance of two or more models on a single test dataset."),ke=c(),u=r("p"),dt=s("For instance, the "),O=r("a"),vt=s("McNemar Test"),gt=s(" is a paired nonparametric statistical hypothesis test that takes the predictions of two models and compares them, aiming to measure whether the models\u2019s predictions diverge or not. The p value it outputs, which ranges from "),ue=r("code"),wt=s("0.0"),yt=s(" to "),me=r("code"),_t=s("1.0"),Et=s(", indicates the difference between the two models\u2019 predictions, with a lower p value indicating a more significant difference."),Pe=c(),ee=r("p"),bt=s("Comparisons have yet to be systematically used when comparing and reporting model performance, however they are useful tools to go beyond simply comparing leaderboard scores and for getting more information on the way model prediction differ."),Te=c(),$=r("h2"),P=r("a"),de=r("span"),Re(R.$$.fragment),$t=c(),ve=r("span"),xt=s("Measurements"),Me=c(),te=r("p"),At=s("In the \u{1F917} Evaluate library, measurements are tools for gaining more insights on datasets and model predictions."),Ie=c(),g=r("p"),kt=s("For instance, in the case of datasets, it can be useful to calculate the "),F=r("a"),Pt=s("average word length"),Tt=s(" of a dataset\u2019s entries, and how it is distributed \u2014 this can help when choosing the maximum input length for "),U=r("a"),Mt=s("Tokenizer"),It=s("."),Se=c(),m=r("p"),St=s("In the case of model predictions, it can help to calculate the average "),z=r("a"),Ct=s("perplexity"),Nt=s(" of model predictions using different models such as "),B=r("a"),Lt=s("GPT-2"),Ht=s(" and "),D=r("a"),qt=s("BERT"),Gt=s(", which can indicate the quality of generated text when no reference is available."),Ce=c(),ae=r("p"),Ot=s("All three types of evaluation supported by the \u{1F917} Evaluate library are meant to be mutually complementary, and help our community carry out more mindful and responsible evaluation."),Ne=c(),w=r("p"),Rt=s("We will continue adding more types of metrics, measurements and comparisons in coming months, and are counting on community involvement (via "),W=r("a"),Ft=s("PRs"),Ut=s(" and "),J=r("a"),zt=s("issues"),Bt=s(") to make the library as extensive and inclusive as possible!"),this.h()},l(e){const h=Sa('[data-svelte="svelte-1phssyn"]',document.head);y=o(h,"META",{name:!0,content:!0}),h.forEach(t),ge=p(e),_=o(e,"H1",{class:!0});var He=n(_);x=o(He,"A",{id:!0,class:!0,href:!0});var Kt=n(x);ie=o(Kt,"SPAN",{});var Qt=n(ie);Fe(I.$$.fragment,Qt),Qt.forEach(t),Kt.forEach(t),Je=p(He),le=o(He,"SPAN",{});var Vt=n(le);je=i(Vt,"Types of Evaluations in \u{1F917} Evaluate"),Vt.forEach(t),He.forEach(t),we=p(e),j=o(e,"P",{});var Xt=n(j);Ke=i(Xt,"The goal of the \u{1F917} Evaluate library is to support different types of evaluation, depending on different goals, datasets and models."),Xt.forEach(t),ye=p(e),K=o(e,"P",{});var Yt=n(K);Qe=i(Yt,"Here are the types of evaluations that are currently supported with a few examples for each:"),Yt.forEach(t),_e=p(e),E=o(e,"H2",{class:!0});var qe=n(E);A=o(qe,"A",{id:!0,class:!0,href:!0});var Zt=n(A);he=o(Zt,"SPAN",{});var ea=n(he);Fe(S.$$.fragment,ea),ea.forEach(t),Zt.forEach(t),Ve=p(qe),fe=o(qe,"SPAN",{});var ta=n(fe);Xe=i(ta,"Metrics"),ta.forEach(t),qe.forEach(t),Ee=i(e,`

A metric measures the performance of a model on a given dataset. This is often based on an existing ground truth (i.e. a set of references), but there are also *referenceless metrics* which allow evaluating generated text by leveraging a pretrained model such as [GPT-2](https://huggingface.co/gpt2).
`),Q=o(e,"P",{});var aa=n(Q);Ye=i(aa,"Examples of metrics include:"),aa.forEach(t),be=p(e),d=o(e,"UL",{});var re=n(d);V=o(re,"LI",{});var Dt=n(V);C=o(Dt,"A",{href:!0,rel:!0});var ra=n(C);Ze=i(ra,"Accuracy"),ra.forEach(t),et=i(Dt," : the proportion of correct predictions among the total number of cases processed."),Dt.forEach(t),tt=p(re),X=o(re,"LI",{});var Wt=n(X);N=o(Wt,"A",{href:!0,rel:!0});var oa=n(N);at=i(oa,"Exact Match"),oa.forEach(t),rt=i(Wt,": the rate at which the input predicted strings exactly match their references."),Wt.forEach(t),ot=p(re),Y=o(re,"LI",{});var Jt=n(Y);L=o(Jt,"A",{href:!0,rel:!0});var sa=n(L);st=i(sa,"Mean Intersection over union (IoUO)"),sa.forEach(t),nt=i(Jt,": the area of overlap between the predicted segmentation of an image and the ground truth divided by the area of union between the predicted segmentation and the ground truth."),Jt.forEach(t),re.forEach(t),$e=p(e),v=o(e,"P",{});var oe=n(v);it=i(oe,"Metrics are often used to track model performance on benchmark datasets, and to report progress on tasks such as "),H=o(oe,"A",{href:!0,rel:!0});var na=n(H);lt=i(na,"machine translation"),na.forEach(t),ht=i(oe," and "),q=o(oe,"A",{href:!0,rel:!0});var ia=n(q);ft=i(ia,"image classification"),ia.forEach(t),ct=i(oe,"."),oe.forEach(t),xe=p(e),b=o(e,"H2",{class:!0});var Ge=n(b);k=o(Ge,"A",{id:!0,class:!0,href:!0});var la=n(k);ce=o(la,"SPAN",{});var ha=n(ce);Fe(G.$$.fragment,ha),ha.forEach(t),la.forEach(t),pt=p(Ge),pe=o(Ge,"SPAN",{});var fa=n(pe);ut=i(fa,"Comparisons"),fa.forEach(t),Ge.forEach(t),Ae=p(e),Z=o(e,"P",{});var ca=n(Z);mt=i(ca,"Comparisons can be useful to compare the performance of two or more models on a single test dataset."),ca.forEach(t),ke=p(e),u=o(e,"P",{});var T=n(u);dt=i(T,"For instance, the "),O=o(T,"A",{href:!0,rel:!0});var pa=n(O);vt=i(pa,"McNemar Test"),pa.forEach(t),gt=i(T," is a paired nonparametric statistical hypothesis test that takes the predictions of two models and compares them, aiming to measure whether the models\u2019s predictions diverge or not. The p value it outputs, which ranges from "),ue=o(T,"CODE",{});var ua=n(ue);wt=i(ua,"0.0"),ua.forEach(t),yt=i(T," to "),me=o(T,"CODE",{});var ma=n(me);_t=i(ma,"1.0"),ma.forEach(t),Et=i(T,", indicates the difference between the two models\u2019 predictions, with a lower p value indicating a more significant difference."),T.forEach(t),Pe=p(e),ee=o(e,"P",{});var da=n(ee);bt=i(da,"Comparisons have yet to be systematically used when comparing and reporting model performance, however they are useful tools to go beyond simply comparing leaderboard scores and for getting more information on the way model prediction differ."),da.forEach(t),Te=p(e),$=o(e,"H2",{class:!0});var Oe=n($);P=o(Oe,"A",{id:!0,class:!0,href:!0});var va=n(P);de=o(va,"SPAN",{});var ga=n(de);Fe(R.$$.fragment,ga),ga.forEach(t),va.forEach(t),$t=p(Oe),ve=o(Oe,"SPAN",{});var wa=n(ve);xt=i(wa,"Measurements"),wa.forEach(t),Oe.forEach(t),Me=p(e),te=o(e,"P",{});var ya=n(te);At=i(ya,"In the \u{1F917} Evaluate library, measurements are tools for gaining more insights on datasets and model predictions."),ya.forEach(t),Ie=p(e),g=o(e,"P",{});var se=n(g);kt=i(se,"For instance, in the case of datasets, it can be useful to calculate the "),F=o(se,"A",{href:!0,rel:!0});var _a=n(F);Pt=i(_a,"average word length"),_a.forEach(t),Tt=i(se," of a dataset\u2019s entries, and how it is distributed \u2014 this can help when choosing the maximum input length for "),U=o(se,"A",{href:!0,rel:!0});var Ea=n(U);Mt=i(Ea,"Tokenizer"),Ea.forEach(t),It=i(se,"."),se.forEach(t),Se=p(e),m=o(e,"P",{});var M=n(m);St=i(M,"In the case of model predictions, it can help to calculate the average "),z=o(M,"A",{href:!0,rel:!0});var ba=n(z);Ct=i(ba,"perplexity"),ba.forEach(t),Nt=i(M," of model predictions using different models such as "),B=o(M,"A",{href:!0,rel:!0});var $a=n(B);Lt=i($a,"GPT-2"),$a.forEach(t),Ht=i(M," and "),D=o(M,"A",{href:!0,rel:!0});var xa=n(D);qt=i(xa,"BERT"),xa.forEach(t),Gt=i(M,", which can indicate the quality of generated text when no reference is available."),M.forEach(t),Ce=p(e),ae=o(e,"P",{});var Aa=n(ae);Ot=i(Aa,"All three types of evaluation supported by the \u{1F917} Evaluate library are meant to be mutually complementary, and help our community carry out more mindful and responsible evaluation."),Aa.forEach(t),Ne=p(e),w=o(e,"P",{});var ne=n(w);Rt=i(ne,"We will continue adding more types of metrics, measurements and comparisons in coming months, and are counting on community involvement (via "),W=o(ne,"A",{href:!0,rel:!0});var ka=n(W);Ft=i(ka,"PRs"),ka.forEach(t),Ut=i(ne," and "),J=o(ne,"A",{href:!0,rel:!0});var Pa=n(J);zt=i(Pa,"issues"),Pa.forEach(t),Bt=i(ne,") to make the library as extensive and inclusive as possible!"),ne.forEach(t),this.h()},h(){l(y,"name","hf:doc:metadata"),l(y,"content",JSON.stringify(Ha)),l(x,"id","types-of-evaluations-in-evaluate"),l(x,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),l(x,"href","#types-of-evaluations-in-evaluate"),l(_,"class","relative group"),l(A,"id","metrics"),l(A,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),l(A,"href","#metrics"),l(E,"class","relative group"),l(C,"href","https://huggingface.co/metrics/accuracy"),l(C,"rel","nofollow"),l(N,"href","https://huggingface.co/metrics/exact_match"),l(N,"rel","nofollow"),l(L,"href","https://huggingface.co/metrics/mean_iou"),l(L,"rel","nofollow"),l(H,"href","https://huggingface.co/tasks/translation"),l(H,"rel","nofollow"),l(q,"href","https://huggingface.co/tasks/image-classification"),l(q,"rel","nofollow"),l(k,"id","comparisons"),l(k,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),l(k,"href","#comparisons"),l(b,"class","relative group"),l(O,"href","https://github.com/huggingface/evaluate/tree/main/comparisons/mcnemar"),l(O,"rel","nofollow"),l(P,"id","measurements"),l(P,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),l(P,"href","#measurements"),l($,"class","relative group"),l(F,"href","https://github.com/huggingface/evaluate/tree/main/measurements/word_length"),l(F,"rel","nofollow"),l(U,"href","https://huggingface.co/docs/transformers/main_classes/tokenizer"),l(U,"rel","nofollow"),l(z,"href","https://huggingface.co/metrics/perplexity"),l(z,"rel","nofollow"),l(B,"href","https://huggingface.co/gpt2"),l(B,"rel","nofollow"),l(D,"href","https://huggingface.co/bert-base-uncased"),l(D,"rel","nofollow"),l(W,"href","https://github.com/huggingface/evaluate/compare"),l(W,"rel","nofollow"),l(J,"href","https://github.com/huggingface/evaluate/issues/new/choose"),l(J,"rel","nofollow")},m(e,h){a(document.head,y),f(e,ge,h),f(e,_,h),a(_,x),a(x,ie),Ue(I,ie,null),a(_,Je),a(_,le),a(le,je),f(e,we,h),f(e,j,h),a(j,Ke),f(e,ye,h),f(e,K,h),a(K,Qe),f(e,_e,h),f(e,E,h),a(E,A),a(A,he),Ue(S,he,null),a(E,Ve),a(E,fe),a(fe,Xe),f(e,Ee,h),f(e,Q,h),a(Q,Ye),f(e,be,h),f(e,d,h),a(d,V),a(V,C),a(C,Ze),a(V,et),a(d,tt),a(d,X),a(X,N),a(N,at),a(X,rt),a(d,ot),a(d,Y),a(Y,L),a(L,st),a(Y,nt),f(e,$e,h),f(e,v,h),a(v,it),a(v,H),a(H,lt),a(v,ht),a(v,q),a(q,ft),a(v,ct),f(e,xe,h),f(e,b,h),a(b,k),a(k,ce),Ue(G,ce,null),a(b,pt),a(b,pe),a(pe,ut),f(e,Ae,h),f(e,Z,h),a(Z,mt),f(e,ke,h),f(e,u,h),a(u,dt),a(u,O),a(O,vt),a(u,gt),a(u,ue),a(ue,wt),a(u,yt),a(u,me),a(me,_t),a(u,Et),f(e,Pe,h),f(e,ee,h),a(ee,bt),f(e,Te,h),f(e,$,h),a($,P),a(P,de),Ue(R,de,null),a($,$t),a($,ve),a(ve,xt),f(e,Me,h),f(e,te,h),a(te,At),f(e,Ie,h),f(e,g,h),a(g,kt),a(g,F),a(F,Pt),a(g,Tt),a(g,U),a(U,Mt),a(g,It),f(e,Se,h),f(e,m,h),a(m,St),a(m,z),a(z,Ct),a(m,Nt),a(m,B),a(B,Lt),a(m,Ht),a(m,D),a(D,qt),a(m,Gt),f(e,Ce,h),f(e,ae,h),a(ae,Ot),f(e,Ne,h),f(e,w,h),a(w,Rt),a(w,W),a(W,Ft),a(w,Ut),a(w,J),a(J,zt),a(w,Bt),Le=!0},p:Ca,i(e){Le||(ze(I.$$.fragment,e),ze(S.$$.fragment,e),ze(G.$$.fragment,e),ze(R.$$.fragment,e),Le=!0)},o(e){Be(I.$$.fragment,e),Be(S.$$.fragment,e),Be(G.$$.fragment,e),Be(R.$$.fragment,e),Le=!1},d(e){t(y),e&&t(ge),e&&t(_),De(I),e&&t(we),e&&t(j),e&&t(ye),e&&t(K),e&&t(_e),e&&t(E),De(S),e&&t(Ee),e&&t(Q),e&&t(be),e&&t(d),e&&t($e),e&&t(v),e&&t(xe),e&&t(b),De(G),e&&t(Ae),e&&t(Z),e&&t(ke),e&&t(u),e&&t(Pe),e&&t(ee),e&&t(Te),e&&t($),De(R),e&&t(Me),e&&t(te),e&&t(Ie),e&&t(g),e&&t(Se),e&&t(m),e&&t(Ce),e&&t(ae),e&&t(Ne),e&&t(w)}}}const Ha={local:"types-of-evaluations-in-evaluate",sections:[{local:"metrics",title:"Metrics"},{local:"comparisons",title:"Comparisons"},{local:"measurements",title:"Measurements"}],title:"Types of Evaluations in \u{1F917} Evaluate"};function qa(jt){return Na(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ra extends Ta{constructor(y){super();Ma(this,y,qa,La,Ia,{})}}export{Ra as default,Ha as metadata};
