import{S as an,i as rn,s as on,e as a,k as h,w as y,t as l,M as ln,c as r,d as s,m as d,a as o,x as E,h as n,b as p,G as e,g as c,y as w,L as nn,q as k,o as $,B as g,v as hn}from"../../../chunks/vendor-hf-doc-builder.js";import{I as ge}from"../../../chunks/IconCopyLink-hf-doc-builder.js";import{C as Pe}from"../../../chunks/CodeBlock-hf-doc-builder.js";function dn(wo){let N,es,P,F,Ie,se,Gs,Se,zs,ts,Y,Js,xe,Ks,Qs,ss,I,U,Me,ae,Vs,Re,Ws,as,C,Xs,re,Zs,ea,qe,ta,sa,rs,_,oe,aa,le,ra,oa,la,S,na,He,ia,ha,Fe,da,ca,pa,Ye,fa,ua,ne,ma,Ue,_a,va,ba,je,Ge,ya,Ea,os,x,G,ze,ie,wa,he,ka,Je,$a,ga,ls,M,z,Ke,de,ja,Qe,Aa,ns,v,Ta,Ve,Ca,Oa,We,Da,La,Xe,Ba,Na,Ze,Pa,Ia,is,ce,hs,J,Sa,et,xa,Ma,ds,R,K,tt,pe,Ra,fe,qa,st,Ha,Fa,cs,j,Ya,ue,at,Ua,Ga,rt,za,Ja,ot,Ka,Qa,ps,me,fs,Ae,Va,us,f,lt,nt,Wa,Xa,it,ht,Za,er,dt,ct,tr,sr,pt,ft,ar,rr,ut,mt,or,lr,_t,vt,nr,ir,bt,yt,hr,dr,Et,wt,cr,pr,kt,$t,fr,ur,gt,jt,mr,_r,At,Tt,vr,br,Ct,Ot,yr,Er,Dt,Lt,wr,kr,Bt,Nt,$r,gr,m,Pt,jr,Ar,_e,Tr,Cr,It,Or,Dr,St,Lr,Br,xt,Nr,Pr,ve,Mt,Ir,Sr,ms,Te,xr,_s,be,vs,q,Q,Rt,ye,Mr,qt,Rr,bs,V,qr,Ht,Hr,Fr,ys,A,Yr,Ft,Ur,Gr,Yt,zr,Jr,Ut,Kr,Qr,Es,Ee,ws,O,Vr,Gt,Wr,Xr,zt,Zr,eo,ks,we,$s,Ce,to,gs,ke,js,D,so,Jt,ao,ro,Kt,oo,lo,As,W,no,Qt,io,ho,Ts,H,X,Vt,$e,co,Wt,po,Cs,T,fo,Xt,uo,mo,Zt,_o,vo,Oe,bo,yo,Os;return se=new ge({}),ae=new ge({}),ie=new ge({}),de=new ge({}),ce=new Pe({props:{code:`from transformers import AutoModel

model = AutoModel.from_pretrained("bert-base-uncased")
print(model)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModel

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModel.from_pretrained(<span class="hljs-string">&quot;bert-base-uncased&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(model)
...
          (LayerNorm): LayerNorm((<span class="hljs-number">768</span>,), eps=<span class="hljs-number">1e-12</span>, elementwise_affine=<span class="hljs-literal">True</span>)
          (dropout): Dropout(p=<span class="hljs-number">0.1</span>, inplace=<span class="hljs-literal">False</span>)
        )
      )
      (<span class="hljs-number">11</span>): BertLayer(
        (attention): BertAttention(
          (self): BertSelfAttention(
            (query): Linear(in_features=<span class="hljs-number">768</span>, out_features=<span class="hljs-number">768</span>, bias=<span class="hljs-literal">True</span>)
            (key): Linear(in_features=<span class="hljs-number">768</span>, out_features=<span class="hljs-number">768</span>, bias=<span class="hljs-literal">True</span>)
            (value): Linear(in_features=<span class="hljs-number">768</span>, out_features=<span class="hljs-number">768</span>, bias=<span class="hljs-literal">True</span>)
            (dropout): Dropout(p=<span class="hljs-number">0.1</span>, inplace=<span class="hljs-literal">False</span>)
          )
          (output): BertSelfOutput(
            (dense): Linear(in_features=<span class="hljs-number">768</span>, out_features=<span class="hljs-number">768</span>, bias=<span class="hljs-literal">True</span>)
            (LayerNorm): LayerNorm((<span class="hljs-number">768</span>,), eps=<span class="hljs-number">1e-12</span>, elementwise_affine=<span class="hljs-literal">True</span>)
            (dropout): Dropout(p=<span class="hljs-number">0.1</span>, inplace=<span class="hljs-literal">False</span>)
          )
        )
        (intermediate): BertIntermediate(
          (dense): Linear(in_features=<span class="hljs-number">768</span>, out_features=<span class="hljs-number">3072</span>, bias=<span class="hljs-literal">True</span>)
          (intermediate_act_fn): GELUActivation()
        )
        (output): BertOutput(
          (dense): Linear(in_features=<span class="hljs-number">3072</span>, out_features=<span class="hljs-number">768</span>, bias=<span class="hljs-literal">True</span>)
          (LayerNorm): LayerNorm((<span class="hljs-number">768</span>,), eps=<span class="hljs-number">1e-12</span>, elementwise_affine=<span class="hljs-literal">True</span>)
          (dropout): Dropout(p=<span class="hljs-number">0.1</span>, inplace=<span class="hljs-literal">False</span>)
        )
      )
    )
  )
  (pooler): BertPooler(
    (dense): Linear(in_features=<span class="hljs-number">768</span>, out_features=<span class="hljs-number">768</span>, bias=<span class="hljs-literal">True</span>)
    (activation): Tanh()
  )
)`}}),pe=new ge({}),me=new Pe({props:{code:`import torch
import torch.nn as nn

from ..base import BetterTransformerBaseLayer


class BertLayerBetterTransformer(BetterTransformerBaseLayer):
    def __init__(self, bert_layer, config):
...`,highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">import</span> torch.nn <span class="hljs-keyword">as</span> nn

<span class="hljs-keyword">from</span> ..base <span class="hljs-keyword">import</span> BetterTransformerBaseLayer


<span class="hljs-keyword">class</span> <span class="hljs-title class_">BertLayerBetterTransformer</span>(<span class="hljs-title class_ inherited__">BetterTransformerBaseLayer</span>):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, bert_layer, config</span>):
...`}}),be=new Pe({props:{code:`self.is_last_layer = False
self.validate_bettertransformer()`,highlighted:`self.is_last_layer = <span class="hljs-literal">False</span>
self.validate_bettertransformer()`}}),ye=new ge({}),Ee=new Pe({props:{code:`super().forward_checker()

if hidden_states.is_nested:
    attention_mask = None

if attention_mask is not None:
    # attention mask comes in with values 0 and -inf. we convert to torch.nn.TransformerEncoder style bool mask
    # 0->false->keep this token -inf->true->mask this token
    attention_mask = attention_mask.bool()
    attention_mask = torch.reshape(attention_mask, (attention_mask.shape[0], attention_mask.shape[-1]))
    seqlen = attention_mask.shape[1]
    lengths = torch.sum(~attention_mask, 1)
    if not all([l == seqlen for l in lengths]):
        hidden_states = torch._nested_tensor_from_mask(hidden_states, ~attention_mask)
    attention_mask = None`,highlighted:`<span class="hljs-built_in">super</span>().forward_checker()

<span class="hljs-keyword">if</span> hidden_states.is_nested:
    attention_mask = <span class="hljs-literal">None</span>

<span class="hljs-keyword">if</span> attention_mask <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>:
    <span class="hljs-comment"># attention mask comes in with values 0 and -inf. we convert to torch.nn.TransformerEncoder style bool mask</span>
    <span class="hljs-comment"># 0-&gt;false-&gt;keep this token -inf-&gt;true-&gt;mask this token</span>
    attention_mask = attention_mask.<span class="hljs-built_in">bool</span>()
    attention_mask = torch.reshape(attention_mask, (attention_mask.shape[<span class="hljs-number">0</span>], attention_mask.shape[-<span class="hljs-number">1</span>]))
    seqlen = attention_mask.shape[<span class="hljs-number">1</span>]
    lengths = torch.<span class="hljs-built_in">sum</span>(~attention_mask, <span class="hljs-number">1</span>)
    <span class="hljs-keyword">if</span> <span class="hljs-keyword">not</span> <span class="hljs-built_in">all</span>([l == seqlen <span class="hljs-keyword">for</span> l <span class="hljs-keyword">in</span> lengths]):
        hidden_states = torch._nested_tensor_from_mask(hidden_states, ~attention_mask)
    attention_mask = <span class="hljs-literal">None</span>`}}),we=new Pe({props:{code:`hidden_states = torch._transformer_encoder_layer_fwd(
    hidden_states,
    self.embed_dim,
    self.num_heads,
    self.in_proj_weight,
    self.in_proj_bias,
    self.out_proj_weight,
    self.out_proj_bias,
    self.use_gelu,
    self.norm_first,
    self.norm1_eps,
    self.norm1_weight,
    self.norm1_bias,
    self.norm2_weight,
    self.norm2_bias,
    self.linear1_weight,
    self.linear1_bias,
    self.linear2_weight,
    self.linear2_bias,
    attention_mask,
)`,highlighted:`hidden_states = torch._transformer_encoder_layer_fwd(
    hidden_states,
    self.embed_dim,
    self.num_heads,
    self.in_proj_weight,
    self.in_proj_bias,
    self.out_proj_weight,
    self.out_proj_bias,
    self.use_gelu,
    self.norm_first,
    self.norm1_eps,
    self.norm1_weight,
    self.norm1_bias,
    self.norm2_weight,
    self.norm2_bias,
    self.linear1_weight,
    self.linear1_bias,
    self.linear2_weight,
    self.linear2_bias,
    attention_mask,
)`}}),ke=new Pe({props:{code:`if hidden_states.is_nested and self.is_last_layer:
    hidden_states = hidden_states.to_padded_tensor(0.0)
return (hidden_states,)`,highlighted:`<span class="hljs-keyword">if</span> hidden_states.is_nested <span class="hljs-keyword">and</span> self.is_last_layer:
    hidden_states = hidden_states.to_padded_tensor(<span class="hljs-number">0.0</span>)
<span class="hljs-keyword">return</span> (hidden_states,)`}}),$e=new ge({}),{c(){N=a("meta"),es=h(),P=a("h1"),F=a("a"),Ie=a("span"),y(se.$$.fragment),Gs=h(),Se=a("span"),zs=l("Adding BetterTransformer support for new architectures"),ts=h(),Y=a("p"),Js=l("You want to add a new model for "),xe=a("code"),Ks=l("BetterTransformer"),Qs=l(" API? Check this guideline!"),ss=h(),I=a("h2"),U=a("a"),Me=a("span"),y(ae.$$.fragment),Vs=h(),Re=a("span"),Ws=l("Models that should be supported"),as=h(),C=a("p"),Xs=l("In theory, any model that has a transformer encoder layer, similar to the classic encoder described in the "),re=a("a"),Zs=l("\u201CAttention Is All You Need\u201D"),ea=l(` paper should be supported.
More specifically, a model that has an encoder block with a MultiHead-Attention module (with pre or post-attention layer norm) should be convertible to its `),qe=a("code"),ta=l("BetterTransformer"),sa=l(" equivalent. The conditions can be summarized as follows:"),rs=h(),_=a("ul"),oe=a("li"),aa=l("Use classic Multi Head attention module (for example, "),le=a("a"),ra=l("DeBERTa"),oa=l(" cannot be supported)"),la=h(),S=a("li"),na=l("Use either "),He=a("code"),ia=l("gelu"),ha=l(" or "),Fe=a("code"),da=l("relu"),ca=l(" activation function"),pa=h(),Ye=a("li"),fa=l("Have an even number of attention heads"),ua=h(),ne=a("li"),ma=l("Do not use any attention bias (for eg "),Ue=a("code"),_a=l("T5"),va=l(" uses attention bias, therefore cannot be supported)"),ba=h(),je=a("li"),Ge=a("code"),ya=l("eps"),Ea=l(" must be equal between the first and second layer norms for each layer"),os=h(),x=a("h2"),G=a("a"),ze=a("span"),y(ie.$$.fragment),wa=h(),he=a("span"),ka=l("How to convert a model into its "),Je=a("code"),$a=l("BetterTransformer"),ga=l(" format?"),ls=h(),M=a("h3"),z=a("a"),Ke=a("span"),y(de.$$.fragment),ja=h(),Qe=a("span"),Aa=l("Step 1: Identifying the source layer to change"),ns=h(),v=a("p"),Ta=l("First, go to "),Ve=a("code"),Ca=l("optimum/bettertransformer/__init__.py"),Oa=l(" and you\u2019ll see the dictionary "),We=a("code"),Da=l("BETTER_TRANFORMER_LAYERS_MAPPING_DICT"),La=l(". This should contain the mapping between the Module that can be converted to its "),Xe=a("code"),Ba=l("BetterTransformer"),Na=l(` equivalent.
Let us try to do it step by step for `),Ze=a("code"),Pa=l("Bert"),Ia=l(", first we need to identify the layers that needs to be replaced:"),is=h(),y(ce.$$.fragment),hs=h(),J=a("p"),Sa=l("You can clearly see that the layers that needs to be replaced are the "),et=a("code"),xa=l("BertLayer"),Ma=l(" modules since it contains the whole encoder layer module."),ds=h(),R=a("h3"),K=a("a"),tt=a("span"),y(pe.$$.fragment),Ra=h(),fe=a("span"),qa=l("Step 2: Building the "),st=a("code"),Ha=l("xxxLayerBetterTransformer"),Fa=l(" module"),cs=h(),j=a("p"),Ya=l("Check that the identified module is not already copied from another module (by inspecting the source code in "),ue=a("a"),at=a("code"),Ua=l("transformers"),Ga=l(" and checking that the class definition does not start with "),rt=a("code"),za=l("# Copied from ..."),Ja=l(") - and if not, create a class in "),ot=a("code"),Ka=l("bettertransformer/models/encoder_model.py"),Qa=l(`.
Start with those lines:`),ps=h(),y(me.$$.fragment),fs=h(),Ae=a("p"),Va=l("Now, make sure to fill all the necessary attributes, the list of attributes are:"),us=h(),f=a("ul"),lt=a("li"),nt=a("code"),Wa=l("in_proj_weight"),Xa=h(),it=a("li"),ht=a("code"),Za=l("in_proj_bias"),er=h(),dt=a("li"),ct=a("code"),tr=l("out_proj_weight"),sr=h(),pt=a("li"),ft=a("code"),ar=l("out_proj_bias"),rr=h(),ut=a("li"),mt=a("code"),or=l("linear1_weight"),lr=h(),_t=a("li"),vt=a("code"),nr=l("linear1_bias"),ir=h(),bt=a("li"),yt=a("code"),hr=l("linear2_weight"),dr=h(),Et=a("li"),wt=a("code"),cr=l("linear2_bias"),pr=h(),kt=a("li"),$t=a("code"),fr=l("norm1_eps"),ur=h(),gt=a("li"),jt=a("code"),mr=l("norm1_weight"),_r=h(),At=a("li"),Tt=a("code"),vr=l("norm1_bias"),br=h(),Ct=a("li"),Ot=a("code"),yr=l("norm2_weight"),Er=h(),Dt=a("li"),Lt=a("code"),wr=l("norm2_bias"),kr=h(),Bt=a("li"),Nt=a("code"),$r=l("num_heads"),gr=h(),m=a("li"),Pt=a("code"),jr=l("embed_dim"),Ar=l(`
Note that these attributes correspond to all the components that are necessary to run a Transformer Encoder module, check the figure 1 on the `),_e=a("a"),Tr=l("\u201CAttention Is All You Need\u201D"),Cr=l(` paper.
Once you filled all these attributes (sometimes the `),It=a("code"),Or=l("query"),Dr=l(", "),St=a("code"),Lr=l("key"),Br=l(" and "),xt=a("code"),Nr=l("value"),Pr=l(" layers needs to be \u201Ccontigufied\u201D, check the "),ve=a("a"),Mt=a("code"),Ir=l("modeling_encoder.py"),Sr=l(" file to understand more.)"),ms=h(),Te=a("p"),xr=l("Make sure also to add the lines:"),_s=h(),y(be.$$.fragment),vs=h(),q=a("h3"),Q=a("a"),Rt=a("span"),y(ye.$$.fragment),Mr=h(),qt=a("span"),Rr=l("Step 3: Building the forward pass"),bs=h(),V=a("p"),qr=l("First of all, start with the line "),Ht=a("code"),Hr=l("super().forward_checker()"),Fr=l(", this is needed so that the parent class can run all the safety checkers before."),ys=h(),A=a("p"),Yr=l("After the first forward pass, the hidden states needs to be "),Ft=a("em"),Ur=l("nested"),Gr=l(" using the attention mask. Once they are nested, the attention mask is not needed anymore, therefore can be set to "),Yt=a("code"),zr=l("None"),Jr=l(". This is how the forward pass is built for "),Ut=a("code"),Kr=l("Bert"),Qr=l(", these lines should remain pretty much similar accross models, but sometimes the shapes of the attention masks are different across models."),Es=h(),y(Ee.$$.fragment),ws=h(),O=a("p"),Vr=l("Once the "),Gt=a("code"),Wr=l("hidden_states"),Xr=l(" are nested, call "),zt=a("code"),Zr=l("torch._transformer_encoder_layer_fwd"),eo=l(" using the right arguments as follows:"),ks=h(),y(we.$$.fragment),$s=h(),Ce=a("p"),to=l("At the last layer, it is important to \u201Cun-nest\u201D the hidden_states so that it can be processed by the next modules, this is done in these lines:"),gs=h(),y(ke.$$.fragment),js=h(),D=a("p"),so=l("Also make sure to return a "),Jt=a("code"),ao=l("tuple"),ro=l(" to follow the convention of "),Kt=a("code"),oo=l("transformers"),lo=l("."),As=h(),W=a("p"),no=l("The best way to reproduce this experiment on your own model is to try it by get some inspiration from the provided modeling scripts. Of course, we will be happy to help you converting your model if you open an issue or a Pull Request on "),Qt=a("code"),io=l("optimum"),ho=l("!"),Ts=h(),H=a("h3"),X=a("a"),Vt=a("span"),y($e.$$.fragment),co=h(),Wt=a("span"),po=l("Step 4: Sanity check!"),Cs=h(),T=a("p"),fo=l("As a last step, make sure to update the "),Xt=a("code"),uo=l("BETTER_TRANFORMER_LAYERS_MAPPING_DICT"),mo=l(" dictionary in  "),Zt=a("code"),_o=l("optimum/bettertransformer/__init__.py"),vo=l(" with the correct names, and you should be ready to convert your model. Try it out with the conversion method that is presented in the "),Oe=a("a"),bo=l("tutorials sections"),yo=l("!"),this.h()},l(t){const i=ln('[data-svelte="svelte-1phssyn"]',document.head);N=r(i,"META",{name:!0,content:!0}),i.forEach(s),es=d(t),P=r(t,"H1",{class:!0});var Ds=o(P);F=r(Ds,"A",{id:!0,class:!0,href:!0});var ko=o(F);Ie=r(ko,"SPAN",{});var $o=o(Ie);E(se.$$.fragment,$o),$o.forEach(s),ko.forEach(s),Gs=d(Ds),Se=r(Ds,"SPAN",{});var go=o(Se);zs=n(go,"Adding BetterTransformer support for new architectures"),go.forEach(s),Ds.forEach(s),ts=d(t),Y=r(t,"P",{});var Ls=o(Y);Js=n(Ls,"You want to add a new model for "),xe=r(Ls,"CODE",{});var jo=o(xe);Ks=n(jo,"BetterTransformer"),jo.forEach(s),Qs=n(Ls," API? Check this guideline!"),Ls.forEach(s),ss=d(t),I=r(t,"H2",{class:!0});var Bs=o(I);U=r(Bs,"A",{id:!0,class:!0,href:!0});var Ao=o(U);Me=r(Ao,"SPAN",{});var To=o(Me);E(ae.$$.fragment,To),To.forEach(s),Ao.forEach(s),Vs=d(Bs),Re=r(Bs,"SPAN",{});var Co=o(Re);Ws=n(Co,"Models that should be supported"),Co.forEach(s),Bs.forEach(s),as=d(t),C=r(t,"P",{});var De=o(C);Xs=n(De,"In theory, any model that has a transformer encoder layer, similar to the classic encoder described in the "),re=r(De,"A",{href:!0,rel:!0});var Oo=o(re);Zs=n(Oo,"\u201CAttention Is All You Need\u201D"),Oo.forEach(s),ea=n(De,` paper should be supported.
More specifically, a model that has an encoder block with a MultiHead-Attention module (with pre or post-attention layer norm) should be convertible to its `),qe=r(De,"CODE",{});var Do=o(qe);ta=n(Do,"BetterTransformer"),Do.forEach(s),sa=n(De," equivalent. The conditions can be summarized as follows:"),De.forEach(s),rs=d(t),_=r(t,"UL",{});var L=o(_);oe=r(L,"LI",{});var Ns=o(oe);aa=n(Ns,"Use classic Multi Head attention module (for example, "),le=r(Ns,"A",{href:!0,rel:!0});var Lo=o(le);ra=n(Lo,"DeBERTa"),Lo.forEach(s),oa=n(Ns," cannot be supported)"),Ns.forEach(s),la=d(L),S=r(L,"LI",{});var Le=o(S);na=n(Le,"Use either "),He=r(Le,"CODE",{});var Bo=o(He);ia=n(Bo,"gelu"),Bo.forEach(s),ha=n(Le," or "),Fe=r(Le,"CODE",{});var No=o(Fe);da=n(No,"relu"),No.forEach(s),ca=n(Le," activation function"),Le.forEach(s),pa=d(L),Ye=r(L,"LI",{});var Po=o(Ye);fa=n(Po,"Have an even number of attention heads"),Po.forEach(s),ua=d(L),ne=r(L,"LI",{});var Ps=o(ne);ma=n(Ps,"Do not use any attention bias (for eg "),Ue=r(Ps,"CODE",{});var Io=o(Ue);_a=n(Io,"T5"),Io.forEach(s),va=n(Ps," uses attention bias, therefore cannot be supported)"),Ps.forEach(s),ba=d(L),je=r(L,"LI",{});var Eo=o(je);Ge=r(Eo,"CODE",{});var So=o(Ge);ya=n(So,"eps"),So.forEach(s),Ea=n(Eo," must be equal between the first and second layer norms for each layer"),Eo.forEach(s),L.forEach(s),os=d(t),x=r(t,"H2",{class:!0});var Is=o(x);G=r(Is,"A",{id:!0,class:!0,href:!0});var xo=o(G);ze=r(xo,"SPAN",{});var Mo=o(ze);E(ie.$$.fragment,Mo),Mo.forEach(s),xo.forEach(s),wa=d(Is),he=r(Is,"SPAN",{});var Ss=o(he);ka=n(Ss,"How to convert a model into its "),Je=r(Ss,"CODE",{});var Ro=o(Je);$a=n(Ro,"BetterTransformer"),Ro.forEach(s),ga=n(Ss," format?"),Ss.forEach(s),Is.forEach(s),ls=d(t),M=r(t,"H3",{class:!0});var xs=o(M);z=r(xs,"A",{id:!0,class:!0,href:!0});var qo=o(z);Ke=r(qo,"SPAN",{});var Ho=o(Ke);E(de.$$.fragment,Ho),Ho.forEach(s),qo.forEach(s),ja=d(xs),Qe=r(xs,"SPAN",{});var Fo=o(Qe);Aa=n(Fo,"Step 1: Identifying the source layer to change"),Fo.forEach(s),xs.forEach(s),ns=d(t),v=r(t,"P",{});var B=o(v);Ta=n(B,"First, go to "),Ve=r(B,"CODE",{});var Yo=o(Ve);Ca=n(Yo,"optimum/bettertransformer/__init__.py"),Yo.forEach(s),Oa=n(B," and you\u2019ll see the dictionary "),We=r(B,"CODE",{});var Uo=o(We);Da=n(Uo,"BETTER_TRANFORMER_LAYERS_MAPPING_DICT"),Uo.forEach(s),La=n(B,". This should contain the mapping between the Module that can be converted to its "),Xe=r(B,"CODE",{});var Go=o(Xe);Ba=n(Go,"BetterTransformer"),Go.forEach(s),Na=n(B,` equivalent.
Let us try to do it step by step for `),Ze=r(B,"CODE",{});var zo=o(Ze);Pa=n(zo,"Bert"),zo.forEach(s),Ia=n(B,", first we need to identify the layers that needs to be replaced:"),B.forEach(s),is=d(t),E(ce.$$.fragment,t),hs=d(t),J=r(t,"P",{});var Ms=o(J);Sa=n(Ms,"You can clearly see that the layers that needs to be replaced are the "),et=r(Ms,"CODE",{});var Jo=o(et);xa=n(Jo,"BertLayer"),Jo.forEach(s),Ma=n(Ms," modules since it contains the whole encoder layer module."),Ms.forEach(s),ds=d(t),R=r(t,"H3",{class:!0});var Rs=o(R);K=r(Rs,"A",{id:!0,class:!0,href:!0});var Ko=o(K);tt=r(Ko,"SPAN",{});var Qo=o(tt);E(pe.$$.fragment,Qo),Qo.forEach(s),Ko.forEach(s),Ra=d(Rs),fe=r(Rs,"SPAN",{});var qs=o(fe);qa=n(qs,"Step 2: Building the "),st=r(qs,"CODE",{});var Vo=o(st);Ha=n(Vo,"xxxLayerBetterTransformer"),Vo.forEach(s),Fa=n(qs," module"),qs.forEach(s),Rs.forEach(s),cs=d(t),j=r(t,"P",{});var Z=o(j);Ya=n(Z,"Check that the identified module is not already copied from another module (by inspecting the source code in "),ue=r(Z,"A",{href:!0,rel:!0});var Wo=o(ue);at=r(Wo,"CODE",{});var Xo=o(at);Ua=n(Xo,"transformers"),Xo.forEach(s),Wo.forEach(s),Ga=n(Z," and checking that the class definition does not start with "),rt=r(Z,"CODE",{});var Zo=o(rt);za=n(Zo,"# Copied from ..."),Zo.forEach(s),Ja=n(Z,") - and if not, create a class in "),ot=r(Z,"CODE",{});var el=o(ot);Ka=n(el,"bettertransformer/models/encoder_model.py"),el.forEach(s),Qa=n(Z,`.
Start with those lines:`),Z.forEach(s),ps=d(t),E(me.$$.fragment,t),fs=d(t),Ae=r(t,"P",{});var tl=o(Ae);Va=n(tl,"Now, make sure to fill all the necessary attributes, the list of attributes are:"),tl.forEach(s),us=d(t),f=r(t,"UL",{});var u=o(f);lt=r(u,"LI",{});var sl=o(lt);nt=r(sl,"CODE",{});var al=o(nt);Wa=n(al,"in_proj_weight"),al.forEach(s),sl.forEach(s),Xa=d(u),it=r(u,"LI",{});var rl=o(it);ht=r(rl,"CODE",{});var ol=o(ht);Za=n(ol,"in_proj_bias"),ol.forEach(s),rl.forEach(s),er=d(u),dt=r(u,"LI",{});var ll=o(dt);ct=r(ll,"CODE",{});var nl=o(ct);tr=n(nl,"out_proj_weight"),nl.forEach(s),ll.forEach(s),sr=d(u),pt=r(u,"LI",{});var il=o(pt);ft=r(il,"CODE",{});var hl=o(ft);ar=n(hl,"out_proj_bias"),hl.forEach(s),il.forEach(s),rr=d(u),ut=r(u,"LI",{});var dl=o(ut);mt=r(dl,"CODE",{});var cl=o(mt);or=n(cl,"linear1_weight"),cl.forEach(s),dl.forEach(s),lr=d(u),_t=r(u,"LI",{});var pl=o(_t);vt=r(pl,"CODE",{});var fl=o(vt);nr=n(fl,"linear1_bias"),fl.forEach(s),pl.forEach(s),ir=d(u),bt=r(u,"LI",{});var ul=o(bt);yt=r(ul,"CODE",{});var ml=o(yt);hr=n(ml,"linear2_weight"),ml.forEach(s),ul.forEach(s),dr=d(u),Et=r(u,"LI",{});var _l=o(Et);wt=r(_l,"CODE",{});var vl=o(wt);cr=n(vl,"linear2_bias"),vl.forEach(s),_l.forEach(s),pr=d(u),kt=r(u,"LI",{});var bl=o(kt);$t=r(bl,"CODE",{});var yl=o($t);fr=n(yl,"norm1_eps"),yl.forEach(s),bl.forEach(s),ur=d(u),gt=r(u,"LI",{});var El=o(gt);jt=r(El,"CODE",{});var wl=o(jt);mr=n(wl,"norm1_weight"),wl.forEach(s),El.forEach(s),_r=d(u),At=r(u,"LI",{});var kl=o(At);Tt=r(kl,"CODE",{});var $l=o(Tt);vr=n($l,"norm1_bias"),$l.forEach(s),kl.forEach(s),br=d(u),Ct=r(u,"LI",{});var gl=o(Ct);Ot=r(gl,"CODE",{});var jl=o(Ot);yr=n(jl,"norm2_weight"),jl.forEach(s),gl.forEach(s),Er=d(u),Dt=r(u,"LI",{});var Al=o(Dt);Lt=r(Al,"CODE",{});var Tl=o(Lt);wr=n(Tl,"norm2_bias"),Tl.forEach(s),Al.forEach(s),kr=d(u),Bt=r(u,"LI",{});var Cl=o(Bt);Nt=r(Cl,"CODE",{});var Ol=o(Nt);$r=n(Ol,"num_heads"),Ol.forEach(s),Cl.forEach(s),gr=d(u),m=r(u,"LI",{});var b=o(m);Pt=r(b,"CODE",{});var Dl=o(Pt);jr=n(Dl,"embed_dim"),Dl.forEach(s),Ar=n(b,`
Note that these attributes correspond to all the components that are necessary to run a Transformer Encoder module, check the figure 1 on the `),_e=r(b,"A",{href:!0,rel:!0});var Ll=o(_e);Tr=n(Ll,"\u201CAttention Is All You Need\u201D"),Ll.forEach(s),Cr=n(b,` paper.
Once you filled all these attributes (sometimes the `),It=r(b,"CODE",{});var Bl=o(It);Or=n(Bl,"query"),Bl.forEach(s),Dr=n(b,", "),St=r(b,"CODE",{});var Nl=o(St);Lr=n(Nl,"key"),Nl.forEach(s),Br=n(b," and "),xt=r(b,"CODE",{});var Pl=o(xt);Nr=n(Pl,"value"),Pl.forEach(s),Pr=n(b," layers needs to be \u201Ccontigufied\u201D, check the "),ve=r(b,"A",{href:!0,rel:!0});var Il=o(ve);Mt=r(Il,"CODE",{});var Sl=o(Mt);Ir=n(Sl,"modeling_encoder.py"),Sl.forEach(s),Il.forEach(s),Sr=n(b," file to understand more.)"),b.forEach(s),u.forEach(s),ms=d(t),Te=r(t,"P",{});var xl=o(Te);xr=n(xl,"Make sure also to add the lines:"),xl.forEach(s),_s=d(t),E(be.$$.fragment,t),vs=d(t),q=r(t,"H3",{class:!0});var Hs=o(q);Q=r(Hs,"A",{id:!0,class:!0,href:!0});var Ml=o(Q);Rt=r(Ml,"SPAN",{});var Rl=o(Rt);E(ye.$$.fragment,Rl),Rl.forEach(s),Ml.forEach(s),Mr=d(Hs),qt=r(Hs,"SPAN",{});var ql=o(qt);Rr=n(ql,"Step 3: Building the forward pass"),ql.forEach(s),Hs.forEach(s),bs=d(t),V=r(t,"P",{});var Fs=o(V);qr=n(Fs,"First of all, start with the line "),Ht=r(Fs,"CODE",{});var Hl=o(Ht);Hr=n(Hl,"super().forward_checker()"),Hl.forEach(s),Fr=n(Fs,", this is needed so that the parent class can run all the safety checkers before."),Fs.forEach(s),ys=d(t),A=r(t,"P",{});var ee=o(A);Yr=n(ee,"After the first forward pass, the hidden states needs to be "),Ft=r(ee,"EM",{});var Fl=o(Ft);Ur=n(Fl,"nested"),Fl.forEach(s),Gr=n(ee," using the attention mask. Once they are nested, the attention mask is not needed anymore, therefore can be set to "),Yt=r(ee,"CODE",{});var Yl=o(Yt);zr=n(Yl,"None"),Yl.forEach(s),Jr=n(ee,". This is how the forward pass is built for "),Ut=r(ee,"CODE",{});var Ul=o(Ut);Kr=n(Ul,"Bert"),Ul.forEach(s),Qr=n(ee,", these lines should remain pretty much similar accross models, but sometimes the shapes of the attention masks are different across models."),ee.forEach(s),Es=d(t),E(Ee.$$.fragment,t),ws=d(t),O=r(t,"P",{});var Be=o(O);Vr=n(Be,"Once the "),Gt=r(Be,"CODE",{});var Gl=o(Gt);Wr=n(Gl,"hidden_states"),Gl.forEach(s),Xr=n(Be," are nested, call "),zt=r(Be,"CODE",{});var zl=o(zt);Zr=n(zl,"torch._transformer_encoder_layer_fwd"),zl.forEach(s),eo=n(Be," using the right arguments as follows:"),Be.forEach(s),ks=d(t),E(we.$$.fragment,t),$s=d(t),Ce=r(t,"P",{});var Jl=o(Ce);to=n(Jl,"At the last layer, it is important to \u201Cun-nest\u201D the hidden_states so that it can be processed by the next modules, this is done in these lines:"),Jl.forEach(s),gs=d(t),E(ke.$$.fragment,t),js=d(t),D=r(t,"P",{});var Ne=o(D);so=n(Ne,"Also make sure to return a "),Jt=r(Ne,"CODE",{});var Kl=o(Jt);ao=n(Kl,"tuple"),Kl.forEach(s),ro=n(Ne," to follow the convention of "),Kt=r(Ne,"CODE",{});var Ql=o(Kt);oo=n(Ql,"transformers"),Ql.forEach(s),lo=n(Ne,"."),Ne.forEach(s),As=d(t),W=r(t,"P",{});var Ys=o(W);no=n(Ys,"The best way to reproduce this experiment on your own model is to try it by get some inspiration from the provided modeling scripts. Of course, we will be happy to help you converting your model if you open an issue or a Pull Request on "),Qt=r(Ys,"CODE",{});var Vl=o(Qt);io=n(Vl,"optimum"),Vl.forEach(s),ho=n(Ys,"!"),Ys.forEach(s),Ts=d(t),H=r(t,"H3",{class:!0});var Us=o(H);X=r(Us,"A",{id:!0,class:!0,href:!0});var Wl=o(X);Vt=r(Wl,"SPAN",{});var Xl=o(Vt);E($e.$$.fragment,Xl),Xl.forEach(s),Wl.forEach(s),co=d(Us),Wt=r(Us,"SPAN",{});var Zl=o(Wt);po=n(Zl,"Step 4: Sanity check!"),Zl.forEach(s),Us.forEach(s),Cs=d(t),T=r(t,"P",{});var te=o(T);fo=n(te,"As a last step, make sure to update the "),Xt=r(te,"CODE",{});var en=o(Xt);uo=n(en,"BETTER_TRANFORMER_LAYERS_MAPPING_DICT"),en.forEach(s),mo=n(te," dictionary in  "),Zt=r(te,"CODE",{});var tn=o(Zt);_o=n(tn,"optimum/bettertransformer/__init__.py"),tn.forEach(s),vo=n(te," with the correct names, and you should be ready to convert your model. Try it out with the conversion method that is presented in the "),Oe=r(te,"A",{href:!0});var sn=o(Oe);bo=n(sn,"tutorials sections"),sn.forEach(s),yo=n(te,"!"),te.forEach(s),this.h()},h(){p(N,"name","hf:doc:metadata"),p(N,"content",JSON.stringify(cn)),p(F,"id","adding-bettertransformer-support-for-new-architectures"),p(F,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(F,"href","#adding-bettertransformer-support-for-new-architectures"),p(P,"class","relative group"),p(U,"id","models-that-should-be-supported"),p(U,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(U,"href","#models-that-should-be-supported"),p(I,"class","relative group"),p(re,"href","https://arxiv.org/abs/1706.03762"),p(re,"rel","nofollow"),p(le,"href","https://arxiv.org/abs/2006.03654"),p(le,"rel","nofollow"),p(G,"id","how-to-convert-a-model-into-its-bettertransformer-format"),p(G,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(G,"href","#how-to-convert-a-model-into-its-bettertransformer-format"),p(x,"class","relative group"),p(z,"id","step-1-identifying-the-source-layer-to-change"),p(z,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(z,"href","#step-1-identifying-the-source-layer-to-change"),p(M,"class","relative group"),p(K,"id","step-2-building-the-xxxlayerbettertransformer-module"),p(K,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(K,"href","#step-2-building-the-xxxlayerbettertransformer-module"),p(R,"class","relative group"),p(ue,"href","https://github.com/huggingface/transformers"),p(ue,"rel","nofollow"),p(_e,"href","https://arxiv.org/pdf/1706.03762.pdf"),p(_e,"rel","nofollow"),p(ve,"href","https://github.com/huggingface/optimum/tree/main/optimum/bettertransformer/models/encoder_model.py"),p(ve,"rel","nofollow"),p(Q,"id","step-3-building-the-forward-pass"),p(Q,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(Q,"href","#step-3-building-the-forward-pass"),p(q,"class","relative group"),p(X,"id","step-4-sanity-check"),p(X,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(X,"href","#step-4-sanity-check"),p(H,"class","relative group"),p(Oe,"href","./tutorials/content")},m(t,i){e(document.head,N),c(t,es,i),c(t,P,i),e(P,F),e(F,Ie),w(se,Ie,null),e(P,Gs),e(P,Se),e(Se,zs),c(t,ts,i),c(t,Y,i),e(Y,Js),e(Y,xe),e(xe,Ks),e(Y,Qs),c(t,ss,i),c(t,I,i),e(I,U),e(U,Me),w(ae,Me,null),e(I,Vs),e(I,Re),e(Re,Ws),c(t,as,i),c(t,C,i),e(C,Xs),e(C,re),e(re,Zs),e(C,ea),e(C,qe),e(qe,ta),e(C,sa),c(t,rs,i),c(t,_,i),e(_,oe),e(oe,aa),e(oe,le),e(le,ra),e(oe,oa),e(_,la),e(_,S),e(S,na),e(S,He),e(He,ia),e(S,ha),e(S,Fe),e(Fe,da),e(S,ca),e(_,pa),e(_,Ye),e(Ye,fa),e(_,ua),e(_,ne),e(ne,ma),e(ne,Ue),e(Ue,_a),e(ne,va),e(_,ba),e(_,je),e(je,Ge),e(Ge,ya),e(je,Ea),c(t,os,i),c(t,x,i),e(x,G),e(G,ze),w(ie,ze,null),e(x,wa),e(x,he),e(he,ka),e(he,Je),e(Je,$a),e(he,ga),c(t,ls,i),c(t,M,i),e(M,z),e(z,Ke),w(de,Ke,null),e(M,ja),e(M,Qe),e(Qe,Aa),c(t,ns,i),c(t,v,i),e(v,Ta),e(v,Ve),e(Ve,Ca),e(v,Oa),e(v,We),e(We,Da),e(v,La),e(v,Xe),e(Xe,Ba),e(v,Na),e(v,Ze),e(Ze,Pa),e(v,Ia),c(t,is,i),w(ce,t,i),c(t,hs,i),c(t,J,i),e(J,Sa),e(J,et),e(et,xa),e(J,Ma),c(t,ds,i),c(t,R,i),e(R,K),e(K,tt),w(pe,tt,null),e(R,Ra),e(R,fe),e(fe,qa),e(fe,st),e(st,Ha),e(fe,Fa),c(t,cs,i),c(t,j,i),e(j,Ya),e(j,ue),e(ue,at),e(at,Ua),e(j,Ga),e(j,rt),e(rt,za),e(j,Ja),e(j,ot),e(ot,Ka),e(j,Qa),c(t,ps,i),w(me,t,i),c(t,fs,i),c(t,Ae,i),e(Ae,Va),c(t,us,i),c(t,f,i),e(f,lt),e(lt,nt),e(nt,Wa),e(f,Xa),e(f,it),e(it,ht),e(ht,Za),e(f,er),e(f,dt),e(dt,ct),e(ct,tr),e(f,sr),e(f,pt),e(pt,ft),e(ft,ar),e(f,rr),e(f,ut),e(ut,mt),e(mt,or),e(f,lr),e(f,_t),e(_t,vt),e(vt,nr),e(f,ir),e(f,bt),e(bt,yt),e(yt,hr),e(f,dr),e(f,Et),e(Et,wt),e(wt,cr),e(f,pr),e(f,kt),e(kt,$t),e($t,fr),e(f,ur),e(f,gt),e(gt,jt),e(jt,mr),e(f,_r),e(f,At),e(At,Tt),e(Tt,vr),e(f,br),e(f,Ct),e(Ct,Ot),e(Ot,yr),e(f,Er),e(f,Dt),e(Dt,Lt),e(Lt,wr),e(f,kr),e(f,Bt),e(Bt,Nt),e(Nt,$r),e(f,gr),e(f,m),e(m,Pt),e(Pt,jr),e(m,Ar),e(m,_e),e(_e,Tr),e(m,Cr),e(m,It),e(It,Or),e(m,Dr),e(m,St),e(St,Lr),e(m,Br),e(m,xt),e(xt,Nr),e(m,Pr),e(m,ve),e(ve,Mt),e(Mt,Ir),e(m,Sr),c(t,ms,i),c(t,Te,i),e(Te,xr),c(t,_s,i),w(be,t,i),c(t,vs,i),c(t,q,i),e(q,Q),e(Q,Rt),w(ye,Rt,null),e(q,Mr),e(q,qt),e(qt,Rr),c(t,bs,i),c(t,V,i),e(V,qr),e(V,Ht),e(Ht,Hr),e(V,Fr),c(t,ys,i),c(t,A,i),e(A,Yr),e(A,Ft),e(Ft,Ur),e(A,Gr),e(A,Yt),e(Yt,zr),e(A,Jr),e(A,Ut),e(Ut,Kr),e(A,Qr),c(t,Es,i),w(Ee,t,i),c(t,ws,i),c(t,O,i),e(O,Vr),e(O,Gt),e(Gt,Wr),e(O,Xr),e(O,zt),e(zt,Zr),e(O,eo),c(t,ks,i),w(we,t,i),c(t,$s,i),c(t,Ce,i),e(Ce,to),c(t,gs,i),w(ke,t,i),c(t,js,i),c(t,D,i),e(D,so),e(D,Jt),e(Jt,ao),e(D,ro),e(D,Kt),e(Kt,oo),e(D,lo),c(t,As,i),c(t,W,i),e(W,no),e(W,Qt),e(Qt,io),e(W,ho),c(t,Ts,i),c(t,H,i),e(H,X),e(X,Vt),w($e,Vt,null),e(H,co),e(H,Wt),e(Wt,po),c(t,Cs,i),c(t,T,i),e(T,fo),e(T,Xt),e(Xt,uo),e(T,mo),e(T,Zt),e(Zt,_o),e(T,vo),e(T,Oe),e(Oe,bo),e(T,yo),Os=!0},p:nn,i(t){Os||(k(se.$$.fragment,t),k(ae.$$.fragment,t),k(ie.$$.fragment,t),k(de.$$.fragment,t),k(ce.$$.fragment,t),k(pe.$$.fragment,t),k(me.$$.fragment,t),k(be.$$.fragment,t),k(ye.$$.fragment,t),k(Ee.$$.fragment,t),k(we.$$.fragment,t),k(ke.$$.fragment,t),k($e.$$.fragment,t),Os=!0)},o(t){$(se.$$.fragment,t),$(ae.$$.fragment,t),$(ie.$$.fragment,t),$(de.$$.fragment,t),$(ce.$$.fragment,t),$(pe.$$.fragment,t),$(me.$$.fragment,t),$(be.$$.fragment,t),$(ye.$$.fragment,t),$(Ee.$$.fragment,t),$(we.$$.fragment,t),$(ke.$$.fragment,t),$($e.$$.fragment,t),Os=!1},d(t){s(N),t&&s(es),t&&s(P),g(se),t&&s(ts),t&&s(Y),t&&s(ss),t&&s(I),g(ae),t&&s(as),t&&s(C),t&&s(rs),t&&s(_),t&&s(os),t&&s(x),g(ie),t&&s(ls),t&&s(M),g(de),t&&s(ns),t&&s(v),t&&s(is),g(ce,t),t&&s(hs),t&&s(J),t&&s(ds),t&&s(R),g(pe),t&&s(cs),t&&s(j),t&&s(ps),g(me,t),t&&s(fs),t&&s(Ae),t&&s(us),t&&s(f),t&&s(ms),t&&s(Te),t&&s(_s),g(be,t),t&&s(vs),t&&s(q),g(ye),t&&s(bs),t&&s(V),t&&s(ys),t&&s(A),t&&s(Es),g(Ee,t),t&&s(ws),t&&s(O),t&&s(ks),g(we,t),t&&s($s),t&&s(Ce),t&&s(gs),g(ke,t),t&&s(js),t&&s(D),t&&s(As),t&&s(W),t&&s(Ts),t&&s(H),g($e),t&&s(Cs),t&&s(T)}}}const cn={local:"adding-bettertransformer-support-for-new-architectures",sections:[{local:"models-that-should-be-supported",title:"Models that should be supported"},{local:"how-to-convert-a-model-into-its-bettertransformer-format",sections:[{local:"step-1-identifying-the-source-layer-to-change",title:"Step 1: Identifying the source layer to change"},{local:"step-2-building-the-xxxlayerbettertransformer-module",title:"Step 2: Building the `xxxLayerBetterTransformer` module"},{local:"step-3-building-the-forward-pass",title:"Step 3: Building the forward pass"},{local:"step-4-sanity-check",title:"Step 4: Sanity check!"}],title:"How to convert a model into its `BetterTransformer` format?"}],title:"Adding BetterTransformer support for new architectures"};function pn(wo){return hn(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class _n extends an{constructor(N){super();rn(this,N,pn,dn,on,{})}}export{_n as default,cn as metadata};
