import{S as De,i as Me,s as $e,e as r,k as c,w as Ne,t as s,M as Oe,c as l,d as a,m as u,a as o,x as Ue,h as n,b as i,F as t,g as p,y as Ke,L as je,q as qe,o as He,B as We}from"../chunks/vendor-4833417e.js";import{I as Fe}from"../chunks/IconCopyLink-4b81c553.js";function Je(Q){let v,A,f,d,D,_,V,M,X,O,L,Y,U,m,k,Z,w,ee,te,G,ae,b,re,le,I,oe,T,se,K,y,ne,x,ie,he,j,g,$,fe,pe,N,de,ce,P,ue,B,ve,me,q,E,ge,R,ye,Ee,H;return _=new Fe({}),{c(){v=r("meta"),A=c(),f=r("h1"),d=r("a"),D=r("span"),Ne(_.$$.fragment),V=c(),M=r("span"),X=s("BERTology"),O=c(),L=r("p"),Y=s(`There is a growing field of study concerned with investigating the inner working of large-scale transformers like BERT
(that some call \u201CBERTology\u201D). Some good examples of this field are:`),U=c(),m=r("ul"),k=r("li"),Z=s(`BERT Rediscovers the Classical NLP Pipeline by Ian Tenney, Dipanjan Das, Ellie Pavlick:
`),w=r("a"),ee=s("https://arxiv.org/abs/1905.05950"),te=c(),G=r("li"),ae=s("Are Sixteen Heads Really Better than One? by Paul Michel, Omer Levy, Graham Neubig: "),b=r("a"),re=s("https://arxiv.org/abs/1905.10650"),le=c(),I=r("li"),oe=s(`What Does BERT Look At? An Analysis of BERT\u2019s Attention by Kevin Clark, Urvashi Khandelwal, Omer Levy, Christopher D.
Manning: `),T=r("a"),se=s("https://arxiv.org/abs/1906.04341"),K=c(),y=r("p"),ne=s(`In order to help this new field develop, we have included a few additional features in the BERT/GPT/GPT-2 models to
help people access the inner representations, mainly adapted from the great work of Paul Michel
(`),x=r("a"),ie=s("https://arxiv.org/abs/1905.10650"),he=s("):"),j=c(),g=r("ul"),$=r("li"),fe=s("accessing all the hidden-states of BERT/GPT/GPT-2,"),pe=c(),N=r("li"),de=s("accessing all the attention weights for each head of BERT/GPT/GPT-2,"),ce=c(),P=r("li"),ue=s(`retrieving heads output values and gradients to be able to compute head importance score and prune head as explained
in `),B=r("a"),ve=s("https://arxiv.org/abs/1905.10650"),me=s("."),q=c(),E=r("p"),ge=s("To help you understand and use these features, we have added a specific example script: "),R=r("a"),ye=s("bertology.py"),Ee=s(` while extract information and prune a model pre-trained on
GLUE.`),this.h()},l(e){const h=Oe('[data-svelte="svelte-1phssyn"]',document.head);v=l(h,"META",{name:!0,content:!0}),h.forEach(a),A=u(e),f=l(e,"H1",{class:!0});var W=o(f);d=l(W,"A",{id:!0,class:!0,href:!0});var Te=o(d);D=l(Te,"SPAN",{});var xe=o(D);Ue(_.$$.fragment,xe),xe.forEach(a),Te.forEach(a),V=u(W),M=l(W,"SPAN",{});var Pe=o(M);X=n(Pe,"BERTology"),Pe.forEach(a),W.forEach(a),O=u(e),L=l(e,"P",{});var Be=o(L);Y=n(Be,`There is a growing field of study concerned with investigating the inner working of large-scale transformers like BERT
(that some call \u201CBERTology\u201D). Some good examples of this field are:`),Be.forEach(a),U=u(e),m=l(e,"UL",{});var S=o(m);k=l(S,"LI",{});var _e=o(k);Z=n(_e,`BERT Rediscovers the Classical NLP Pipeline by Ian Tenney, Dipanjan Das, Ellie Pavlick:
`),w=l(_e,"A",{href:!0,rel:!0});var Re=o(w);ee=n(Re,"https://arxiv.org/abs/1905.05950"),Re.forEach(a),_e.forEach(a),te=u(S),G=l(S,"LI",{});var we=o(G);ae=n(we,"Are Sixteen Heads Really Better than One? by Paul Michel, Omer Levy, Graham Neubig: "),b=l(we,"A",{href:!0,rel:!0});var Ae=o(b);re=n(Ae,"https://arxiv.org/abs/1905.10650"),Ae.forEach(a),we.forEach(a),le=u(S),I=l(S,"LI",{});var be=o(I);oe=n(be,`What Does BERT Look At? An Analysis of BERT\u2019s Attention by Kevin Clark, Urvashi Khandelwal, Omer Levy, Christopher D.
Manning: `),T=l(be,"A",{href:!0,rel:!0});var Le=o(T);se=n(Le,"https://arxiv.org/abs/1906.04341"),Le.forEach(a),be.forEach(a),S.forEach(a),K=u(e),y=l(e,"P",{});var F=o(y);ne=n(F,`In order to help this new field develop, we have included a few additional features in the BERT/GPT/GPT-2 models to
help people access the inner representations, mainly adapted from the great work of Paul Michel
(`),x=l(F,"A",{href:!0,rel:!0});var ke=o(x);ie=n(ke,"https://arxiv.org/abs/1905.10650"),ke.forEach(a),he=n(F,"):"),F.forEach(a),j=u(e),g=l(e,"UL",{});var C=o(g);$=l(C,"LI",{});var Ge=o($);fe=n(Ge,"accessing all the hidden-states of BERT/GPT/GPT-2,"),Ge.forEach(a),pe=u(C),N=l(C,"LI",{});var Ie=o(N);de=n(Ie,"accessing all the attention weights for each head of BERT/GPT/GPT-2,"),Ie.forEach(a),ce=u(C),P=l(C,"LI",{});var J=o(P);ue=n(J,`retrieving heads output values and gradients to be able to compute head importance score and prune head as explained
in `),B=l(J,"A",{href:!0,rel:!0});var Se=o(B);ve=n(Se,"https://arxiv.org/abs/1905.10650"),Se.forEach(a),me=n(J,"."),J.forEach(a),C.forEach(a),q=u(e),E=l(e,"P",{});var z=o(E);ge=n(z,"To help you understand and use these features, we have added a specific example script: "),R=l(z,"A",{href:!0,rel:!0});var Ce=o(R);ye=n(Ce,"bertology.py"),Ce.forEach(a),Ee=n(z,` while extract information and prune a model pre-trained on
GLUE.`),z.forEach(a),this.h()},h(){i(v,"name","hf:doc:metadata"),i(v,"content",JSON.stringify(ze)),i(d,"id","bertology"),i(d,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(d,"href","#bertology"),i(f,"class","relative group"),i(w,"href","https://arxiv.org/abs/1905.05950"),i(w,"rel","nofollow"),i(b,"href","https://arxiv.org/abs/1905.10650"),i(b,"rel","nofollow"),i(T,"href","https://arxiv.org/abs/1906.04341"),i(T,"rel","nofollow"),i(x,"href","https://arxiv.org/abs/1905.10650"),i(x,"rel","nofollow"),i(B,"href","https://arxiv.org/abs/1905.10650"),i(B,"rel","nofollow"),i(R,"href","https://github.com/huggingface/transformers/tree/master/examples/research_projects/bertology/run_bertology.py"),i(R,"rel","nofollow")},m(e,h){t(document.head,v),p(e,A,h),p(e,f,h),t(f,d),t(d,D),Ke(_,D,null),t(f,V),t(f,M),t(M,X),p(e,O,h),p(e,L,h),t(L,Y),p(e,U,h),p(e,m,h),t(m,k),t(k,Z),t(k,w),t(w,ee),t(m,te),t(m,G),t(G,ae),t(G,b),t(b,re),t(m,le),t(m,I),t(I,oe),t(I,T),t(T,se),p(e,K,h),p(e,y,h),t(y,ne),t(y,x),t(x,ie),t(y,he),p(e,j,h),p(e,g,h),t(g,$),t($,fe),t(g,pe),t(g,N),t(N,de),t(g,ce),t(g,P),t(P,ue),t(P,B),t(B,ve),t(P,me),p(e,q,h),p(e,E,h),t(E,ge),t(E,R),t(R,ye),t(E,Ee),H=!0},p:je,i(e){H||(qe(_.$$.fragment,e),H=!0)},o(e){He(_.$$.fragment,e),H=!1},d(e){a(v),e&&a(A),e&&a(f),We(_),e&&a(O),e&&a(L),e&&a(U),e&&a(m),e&&a(K),e&&a(y),e&&a(j),e&&a(g),e&&a(q),e&&a(E)}}}const ze={local:"bertology",title:"BERTology"};function Qe(Q,v,A){let{fw:f}=v;return Q.$$set=d=>{"fw"in d&&A(0,f=d.fw)},[f]}class Ye extends De{constructor(v){super();Me(this,v,Qe,Je,$e,{fw:0})}}export{Ye as default,ze as metadata};
