import{S as Me,i as Ce,s as De,e as r,k as p,w as Ne,t as s,M as Oe,c as l,d as a,m as c,a as o,x as Ue,h as n,b as i,G as t,g as f,y as Ke,L as je,q as qe,o as He,B as We,v as Je}from"../chunks/vendor-hf-doc-builder.js";import{I as ze}from"../chunks/IconCopyLink-hf-doc-builder.js";function Fe(be){let v,N,m,g,$,_,Q,M,V,O,L,X,U,d,A,Y,w,Z,ee,k,te,b,ae,re,G,le,T,oe,K,y,se,x,ne,ie,j,u,C,he,fe,D,pe,ce,P,de,R,ue,ve,q,E,me,B,ge,ye,H;return _=new ze({}),{c(){v=r("meta"),N=p(),m=r("h1"),g=r("a"),$=r("span"),Ne(_.$$.fragment),Q=p(),M=r("span"),V=s("BERTology"),O=p(),L=r("p"),X=s(`There is a growing field of study concerned with investigating the inner working of large-scale transformers like BERT
(that some call \u201CBERTology\u201D). Some good examples of this field are:`),U=p(),d=r("ul"),A=r("li"),Y=s(`BERT Rediscovers the Classical NLP Pipeline by Ian Tenney, Dipanjan Das, Ellie Pavlick:
`),w=r("a"),Z=s("https://arxiv.org/abs/1905.05950"),ee=p(),k=r("li"),te=s("Are Sixteen Heads Really Better than One? by Paul Michel, Omer Levy, Graham Neubig: "),b=r("a"),ae=s("https://arxiv.org/abs/1905.10650"),re=p(),G=r("li"),le=s(`What Does BERT Look At? An Analysis of BERT\u2019s Attention by Kevin Clark, Urvashi Khandelwal, Omer Levy, Christopher D.
Manning: `),T=r("a"),oe=s("https://arxiv.org/abs/1906.04341"),K=p(),y=r("p"),se=s(`In order to help this new field develop, we have included a few additional features in the BERT/GPT/GPT-2 models to
help people access the inner representations, mainly adapted from the great work of Paul Michel
(`),x=r("a"),ne=s("https://arxiv.org/abs/1905.10650"),ie=s("):"),j=p(),u=r("ul"),C=r("li"),he=s("accessing all the hidden-states of BERT/GPT/GPT-2,"),fe=p(),D=r("li"),pe=s("accessing all the attention weights for each head of BERT/GPT/GPT-2,"),ce=p(),P=r("li"),de=s(`retrieving heads output values and gradients to be able to compute head importance score and prune head as explained
in `),R=r("a"),ue=s("https://arxiv.org/abs/1905.10650"),ve=s("."),q=p(),E=r("p"),me=s("To help you understand and use these features, we have added a specific example script: "),B=r("a"),ge=s("bertology.py"),ye=s(` while extract information and prune a model pre-trained on
GLUE.`),this.h()},l(e){const h=Oe('[data-svelte="svelte-1phssyn"]',document.head);v=l(h,"META",{name:!0,content:!0}),h.forEach(a),N=c(e),m=l(e,"H1",{class:!0});var W=o(m);g=l(W,"A",{id:!0,class:!0,href:!0});var Te=o(g);$=l(Te,"SPAN",{});var xe=o($);Ue(_.$$.fragment,xe),xe.forEach(a),Te.forEach(a),Q=c(W),M=l(W,"SPAN",{});var Pe=o(M);V=n(Pe,"BERTology"),Pe.forEach(a),W.forEach(a),O=c(e),L=l(e,"P",{});var Re=o(L);X=n(Re,`There is a growing field of study concerned with investigating the inner working of large-scale transformers like BERT
(that some call \u201CBERTology\u201D). Some good examples of this field are:`),Re.forEach(a),U=c(e),d=l(e,"UL",{});var I=o(d);A=l(I,"LI",{});var Ee=o(A);Y=n(Ee,`BERT Rediscovers the Classical NLP Pipeline by Ian Tenney, Dipanjan Das, Ellie Pavlick:
`),w=l(Ee,"A",{href:!0,rel:!0});var Be=o(w);Z=n(Be,"https://arxiv.org/abs/1905.05950"),Be.forEach(a),Ee.forEach(a),ee=c(I),k=l(I,"LI",{});var _e=o(k);te=n(_e,"Are Sixteen Heads Really Better than One? by Paul Michel, Omer Levy, Graham Neubig: "),b=l(_e,"A",{href:!0,rel:!0});var Le=o(b);ae=n(Le,"https://arxiv.org/abs/1905.10650"),Le.forEach(a),_e.forEach(a),re=c(I),G=l(I,"LI",{});var we=o(G);le=n(we,`What Does BERT Look At? An Analysis of BERT\u2019s Attention by Kevin Clark, Urvashi Khandelwal, Omer Levy, Christopher D.
Manning: `),T=l(we,"A",{href:!0,rel:!0});var Ae=o(T);oe=n(Ae,"https://arxiv.org/abs/1906.04341"),Ae.forEach(a),we.forEach(a),I.forEach(a),K=c(e),y=l(e,"P",{});var J=o(y);se=n(J,`In order to help this new field develop, we have included a few additional features in the BERT/GPT/GPT-2 models to
help people access the inner representations, mainly adapted from the great work of Paul Michel
(`),x=l(J,"A",{href:!0,rel:!0});var ke=o(x);ne=n(ke,"https://arxiv.org/abs/1905.10650"),ke.forEach(a),ie=n(J,"):"),J.forEach(a),j=c(e),u=l(e,"UL",{});var S=o(u);C=l(S,"LI",{});var Ge=o(C);he=n(Ge,"accessing all the hidden-states of BERT/GPT/GPT-2,"),Ge.forEach(a),fe=c(S),D=l(S,"LI",{});var Ie=o(D);pe=n(Ie,"accessing all the attention weights for each head of BERT/GPT/GPT-2,"),Ie.forEach(a),ce=c(S),P=l(S,"LI",{});var z=o(P);de=n(z,`retrieving heads output values and gradients to be able to compute head importance score and prune head as explained
in `),R=l(z,"A",{href:!0,rel:!0});var Se=o(R);ue=n(Se,"https://arxiv.org/abs/1905.10650"),Se.forEach(a),ve=n(z,"."),z.forEach(a),S.forEach(a),q=c(e),E=l(e,"P",{});var F=o(E);me=n(F,"To help you understand and use these features, we have added a specific example script: "),B=l(F,"A",{href:!0,rel:!0});var $e=o(B);ge=n($e,"bertology.py"),$e.forEach(a),ye=n(F,` while extract information and prune a model pre-trained on
GLUE.`),F.forEach(a),this.h()},h(){i(v,"name","hf:doc:metadata"),i(v,"content",JSON.stringify(Qe)),i(g,"id","bertology"),i(g,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(g,"href","#bertology"),i(m,"class","relative group"),i(w,"href","https://arxiv.org/abs/1905.05950"),i(w,"rel","nofollow"),i(b,"href","https://arxiv.org/abs/1905.10650"),i(b,"rel","nofollow"),i(T,"href","https://arxiv.org/abs/1906.04341"),i(T,"rel","nofollow"),i(x,"href","https://arxiv.org/abs/1905.10650"),i(x,"rel","nofollow"),i(R,"href","https://arxiv.org/abs/1905.10650"),i(R,"rel","nofollow"),i(B,"href","https://github.com/huggingface/transformers/tree/main/examples/research_projects/bertology/run_bertology.py"),i(B,"rel","nofollow")},m(e,h){t(document.head,v),f(e,N,h),f(e,m,h),t(m,g),t(g,$),Ke(_,$,null),t(m,Q),t(m,M),t(M,V),f(e,O,h),f(e,L,h),t(L,X),f(e,U,h),f(e,d,h),t(d,A),t(A,Y),t(A,w),t(w,Z),t(d,ee),t(d,k),t(k,te),t(k,b),t(b,ae),t(d,re),t(d,G),t(G,le),t(G,T),t(T,oe),f(e,K,h),f(e,y,h),t(y,se),t(y,x),t(x,ne),t(y,ie),f(e,j,h),f(e,u,h),t(u,C),t(C,he),t(u,fe),t(u,D),t(D,pe),t(u,ce),t(u,P),t(P,de),t(P,R),t(R,ue),t(P,ve),f(e,q,h),f(e,E,h),t(E,me),t(E,B),t(B,ge),t(E,ye),H=!0},p:je,i(e){H||(qe(_.$$.fragment,e),H=!0)},o(e){He(_.$$.fragment,e),H=!1},d(e){a(v),e&&a(N),e&&a(m),We(_),e&&a(O),e&&a(L),e&&a(U),e&&a(d),e&&a(K),e&&a(y),e&&a(j),e&&a(u),e&&a(q),e&&a(E)}}}const Qe={local:"bertology",title:"BERTology"};function Ve(be){return Je(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ze extends Me{constructor(v){super();Ce(this,v,Ve,Fe,De,{})}}export{Ze as default,Qe as metadata};
