import{S as Po,i as Ao,s as Lo,e as r,k as p,w as St,t as n,M as Co,c as l,d as o,m as d,a,x as Gt,h as i,b as s,F as e,g as h,y as Nt,L as Io,q as qt,o as Dt,B as Bt}from"../../chunks/vendor-4833417e.js";import{I as $o}from"../../chunks/IconCopyLink-4b81c553.js";import{C as Oo}from"../../chunks/CodeBlock-6a3d1b46.js";import"../../chunks/CopyButton-dacfbfaf.js";function So(De){let _,V,c,m,Q,A,Be,W,Ue,we,x,y,X,L,Fe,Y,Re,Ee,T,Ve,C,je,He,be,j,Me,xe,I,ye,H,Je,Te,u,Z,O,ze,S,Ke,Qe,We,ee,te,Xe,Ye,oe,re,Ze,et,le,ae,tt,ot,ne,g,rt,ie,lt,at,se,nt,it,fe,st,ft,ke,k,ht,G,pt,dt,$e,M,ut,Pe,v,he,pe,N,vt,ct,de,ue,q,mt,gt,ve,ce,D,_t,wt,me,ge,B,Et,bt,_e,J,U,xt,yt,Ae,$,Tt,z,kt,$t,Le,w,Pt,F,At,Lt,R,Ct,It,Ce;return A=new $o({}),L=new $o({}),I=new Oo({props:{code:`from transformers import T5ForConditionalGeneration

model = T5ForConditionalGeneration.from_pretrained("google/t5-v1_1-base")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> T5ForConditionalGeneration

model = T5ForConditionalGeneration.from_pretrained(<span class="hljs-string">&quot;google/t5-v1_1-base&quot;</span>)`}}),{c(){_=r("meta"),V=p(),c=r("h1"),m=r("a"),Q=r("span"),St(A.$$.fragment),Be=p(),W=r("span"),Ue=n("T5v1.1"),we=p(),x=r("h2"),y=r("a"),X=r("span"),St(L.$$.fragment),Fe=p(),Y=r("span"),Re=n("Overview"),Ee=p(),T=r("p"),Ve=n("T5v1.1 was released in the "),C=r("a"),je=n("google-research/text-to-text-transfer-transformer"),He=n(`
repository by Colin Raffel et al. It\u2019s an improved version of the original T5 model.`),be=p(),j=r("p"),Me=n("One can directly plug in the weights of T5v1.1 into a T5 model, like so:"),xe=p(),St(I.$$.fragment),ye=p(),H=r("p"),Je=n("T5 Version 1.1 includes the following improvements compared to the original T5 model:"),Te=p(),u=r("ul"),Z=r("li"),O=r("p"),ze=n("GEGLU activation in the feed-forward hidden layer, rather than ReLU. See "),S=r("a"),Ke=n("this paper"),Qe=n("."),We=p(),ee=r("li"),te=r("p"),Xe=n("Dropout was turned off in pre-training (quality win). Dropout should be re-enabled during fine-tuning."),Ye=p(),oe=r("li"),re=r("p"),Ze=n("Pre-trained on C4 only without mixing in the downstream tasks."),et=p(),le=r("li"),ae=r("p"),tt=n("No parameter sharing between the embedding and classifier layer."),ot=p(),ne=r("li"),g=r("p"),rt=n("\u201Cxl\u201D and \u201Cxxl\u201D replace \u201C3B\u201D and \u201C11B\u201D. The model shapes are a bit different - larger "),ie=r("code"),lt=n("d_model"),at=n(` and smaller
`),se=r("code"),nt=n("num_heads"),it=n(" and "),fe=r("code"),st=n("d_ff"),ft=n("."),ke=p(),k=r("p"),ht=n("Note: T5 Version 1.1 was only pre-trained on "),G=r("a"),pt=n("C4"),dt=n(` excluding any supervised
training. Therefore, this model has to be fine-tuned before it is useable on a downstream task, unlike the original T5
model. Since t5v1.1 was pre-trained unsupervisedly, there\u2019s no real advantage to using a task prefix during single-task
fine-tuning. If you are doing multi-task fine-tuning, you should use a prefix.`),$e=p(),M=r("p"),ut=n("Google has released the following variants:"),Pe=p(),v=r("ul"),he=r("li"),pe=r("p"),N=r("a"),vt=n("google/t5-v1_1-small"),ct=p(),de=r("li"),ue=r("p"),q=r("a"),mt=n("google/t5-v1_1-base"),gt=p(),ve=r("li"),ce=r("p"),D=r("a"),_t=n("google/t5-v1_1-large"),wt=p(),me=r("li"),ge=r("p"),B=r("a"),Et=n("google/t5-v1_1-xl"),bt=p(),_e=r("li"),J=r("p"),U=r("a"),xt=n("google/t5-v1_1-xxl"),yt=n("."),Ae=p(),$=r("p"),Tt=n("One can refer to "),z=r("a"),kt=n("T5\u2019s documentation page"),$t=n(" for all tips, code examples and notebooks."),Le=p(),w=r("p"),Pt=n("This model was contributed by "),F=r("a"),At=n("patrickvonplaten"),Lt=n(`. The original code can be
found `),R=r("a"),Ct=n("here"),It=n("."),this.h()},l(t){const f=Co('[data-svelte="svelte-1phssyn"]',document.head);_=l(f,"META",{name:!0,content:!0}),f.forEach(o),V=d(t),c=l(t,"H1",{class:!0});var Ie=a(c);m=l(Ie,"A",{id:!0,class:!0,href:!0});var Ut=a(m);Q=l(Ut,"SPAN",{});var Ft=a(Q);Gt(A.$$.fragment,Ft),Ft.forEach(o),Ut.forEach(o),Be=d(Ie),W=l(Ie,"SPAN",{});var Rt=a(W);Ue=i(Rt,"T5v1.1"),Rt.forEach(o),Ie.forEach(o),we=d(t),x=l(t,"H2",{class:!0});var Oe=a(x);y=l(Oe,"A",{id:!0,class:!0,href:!0});var Vt=a(y);X=l(Vt,"SPAN",{});var jt=a(X);Gt(L.$$.fragment,jt),jt.forEach(o),Vt.forEach(o),Fe=d(Oe),Y=l(Oe,"SPAN",{});var Ht=a(Y);Re=i(Ht,"Overview"),Ht.forEach(o),Oe.forEach(o),Ee=d(t),T=l(t,"P",{});var Se=a(T);Ve=i(Se,"T5v1.1 was released in the "),C=l(Se,"A",{href:!0,rel:!0});var Mt=a(C);je=i(Mt,"google-research/text-to-text-transfer-transformer"),Mt.forEach(o),He=i(Se,`
repository by Colin Raffel et al. It\u2019s an improved version of the original T5 model.`),Se.forEach(o),be=d(t),j=l(t,"P",{});var Jt=a(j);Me=i(Jt,"One can directly plug in the weights of T5v1.1 into a T5 model, like so:"),Jt.forEach(o),xe=d(t),Gt(I.$$.fragment,t),ye=d(t),H=l(t,"P",{});var zt=a(H);Je=i(zt,"T5 Version 1.1 includes the following improvements compared to the original T5 model:"),zt.forEach(o),Te=d(t),u=l(t,"UL",{});var E=a(u);Z=l(E,"LI",{});var Kt=a(Z);O=l(Kt,"P",{});var Ge=a(O);ze=i(Ge,"GEGLU activation in the feed-forward hidden layer, rather than ReLU. See "),S=l(Ge,"A",{href:!0,rel:!0});var Qt=a(S);Ke=i(Qt,"this paper"),Qt.forEach(o),Qe=i(Ge,"."),Ge.forEach(o),Kt.forEach(o),We=d(E),ee=l(E,"LI",{});var Wt=a(ee);te=l(Wt,"P",{});var Xt=a(te);Xe=i(Xt,"Dropout was turned off in pre-training (quality win). Dropout should be re-enabled during fine-tuning."),Xt.forEach(o),Wt.forEach(o),Ye=d(E),oe=l(E,"LI",{});var Yt=a(oe);re=l(Yt,"P",{});var Zt=a(re);Ze=i(Zt,"Pre-trained on C4 only without mixing in the downstream tasks."),Zt.forEach(o),Yt.forEach(o),et=d(E),le=l(E,"LI",{});var eo=a(le);ae=l(eo,"P",{});var to=a(ae);tt=i(to,"No parameter sharing between the embedding and classifier layer."),to.forEach(o),eo.forEach(o),ot=d(E),ne=l(E,"LI",{});var oo=a(ne);g=l(oo,"P",{});var P=a(g);rt=i(P,"\u201Cxl\u201D and \u201Cxxl\u201D replace \u201C3B\u201D and \u201C11B\u201D. The model shapes are a bit different - larger "),ie=l(P,"CODE",{});var ro=a(ie);lt=i(ro,"d_model"),ro.forEach(o),at=i(P,` and smaller
`),se=l(P,"CODE",{});var lo=a(se);nt=i(lo,"num_heads"),lo.forEach(o),it=i(P," and "),fe=l(P,"CODE",{});var ao=a(fe);st=i(ao,"d_ff"),ao.forEach(o),ft=i(P,"."),P.forEach(o),oo.forEach(o),E.forEach(o),ke=d(t),k=l(t,"P",{});var Ne=a(k);ht=i(Ne,"Note: T5 Version 1.1 was only pre-trained on "),G=l(Ne,"A",{href:!0,rel:!0});var no=a(G);pt=i(no,"C4"),no.forEach(o),dt=i(Ne,` excluding any supervised
training. Therefore, this model has to be fine-tuned before it is useable on a downstream task, unlike the original T5
model. Since t5v1.1 was pre-trained unsupervisedly, there\u2019s no real advantage to using a task prefix during single-task
fine-tuning. If you are doing multi-task fine-tuning, you should use a prefix.`),Ne.forEach(o),$e=d(t),M=l(t,"P",{});var io=a(M);ut=i(io,"Google has released the following variants:"),io.forEach(o),Pe=d(t),v=l(t,"UL",{});var b=a(v);he=l(b,"LI",{});var so=a(he);pe=l(so,"P",{});var fo=a(pe);N=l(fo,"A",{href:!0,rel:!0});var ho=a(N);vt=i(ho,"google/t5-v1_1-small"),ho.forEach(o),fo.forEach(o),so.forEach(o),ct=d(b),de=l(b,"LI",{});var po=a(de);ue=l(po,"P",{});var uo=a(ue);q=l(uo,"A",{href:!0,rel:!0});var vo=a(q);mt=i(vo,"google/t5-v1_1-base"),vo.forEach(o),uo.forEach(o),po.forEach(o),gt=d(b),ve=l(b,"LI",{});var co=a(ve);ce=l(co,"P",{});var mo=a(ce);D=l(mo,"A",{href:!0,rel:!0});var go=a(D);_t=i(go,"google/t5-v1_1-large"),go.forEach(o),mo.forEach(o),co.forEach(o),wt=d(b),me=l(b,"LI",{});var _o=a(me);ge=l(_o,"P",{});var wo=a(ge);B=l(wo,"A",{href:!0,rel:!0});var Eo=a(B);Et=i(Eo,"google/t5-v1_1-xl"),Eo.forEach(o),wo.forEach(o),_o.forEach(o),bt=d(b),_e=l(b,"LI",{});var bo=a(_e);J=l(bo,"P",{});var Ot=a(J);U=l(Ot,"A",{href:!0,rel:!0});var xo=a(U);xt=i(xo,"google/t5-v1_1-xxl"),xo.forEach(o),yt=i(Ot,"."),Ot.forEach(o),bo.forEach(o),b.forEach(o),Ae=d(t),$=l(t,"P",{});var qe=a($);Tt=i(qe,"One can refer to "),z=l(qe,"A",{href:!0});var yo=a(z);kt=i(yo,"T5\u2019s documentation page"),yo.forEach(o),$t=i(qe," for all tips, code examples and notebooks."),qe.forEach(o),Le=d(t),w=l(t,"P",{});var K=a(w);Pt=i(K,"This model was contributed by "),F=l(K,"A",{href:!0,rel:!0});var To=a(F);At=i(To,"patrickvonplaten"),To.forEach(o),Lt=i(K,`. The original code can be
found `),R=l(K,"A",{href:!0,rel:!0});var ko=a(R);Ct=i(ko,"here"),ko.forEach(o),It=i(K,"."),K.forEach(o),this.h()},h(){s(_,"name","hf:doc:metadata"),s(_,"content",JSON.stringify(Go)),s(m,"id","t5v11"),s(m,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),s(m,"href","#t5v11"),s(c,"class","relative group"),s(y,"id","overview"),s(y,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),s(y,"href","#overview"),s(x,"class","relative group"),s(C,"href","https://github.com/google-research/text-to-text-transfer-transformer/blob/main/released_checkpoints.md#t511"),s(C,"rel","nofollow"),s(S,"href","https://arxiv.org/abs/2002.05202"),s(S,"rel","nofollow"),s(G,"href","https://huggingface.co/datasets/c4"),s(G,"rel","nofollow"),s(N,"href","https://huggingface.co/google/t5-v1_1-small"),s(N,"rel","nofollow"),s(q,"href","https://huggingface.co/google/t5-v1_1-base"),s(q,"rel","nofollow"),s(D,"href","https://huggingface.co/google/t5-v1_1-large"),s(D,"rel","nofollow"),s(B,"href","https://huggingface.co/google/t5-v1_1-xl"),s(B,"rel","nofollow"),s(U,"href","https://huggingface.co/google/t5-v1_1-xxl"),s(U,"rel","nofollow"),s(z,"href","t5"),s(F,"href","https://huggingface.co/patrickvonplaten"),s(F,"rel","nofollow"),s(R,"href","https://github.com/google-research/text-to-text-transfer-transformer/blob/main/released_checkpoints.md#t511"),s(R,"rel","nofollow")},m(t,f){e(document.head,_),h(t,V,f),h(t,c,f),e(c,m),e(m,Q),Nt(A,Q,null),e(c,Be),e(c,W),e(W,Ue),h(t,we,f),h(t,x,f),e(x,y),e(y,X),Nt(L,X,null),e(x,Fe),e(x,Y),e(Y,Re),h(t,Ee,f),h(t,T,f),e(T,Ve),e(T,C),e(C,je),e(T,He),h(t,be,f),h(t,j,f),e(j,Me),h(t,xe,f),Nt(I,t,f),h(t,ye,f),h(t,H,f),e(H,Je),h(t,Te,f),h(t,u,f),e(u,Z),e(Z,O),e(O,ze),e(O,S),e(S,Ke),e(O,Qe),e(u,We),e(u,ee),e(ee,te),e(te,Xe),e(u,Ye),e(u,oe),e(oe,re),e(re,Ze),e(u,et),e(u,le),e(le,ae),e(ae,tt),e(u,ot),e(u,ne),e(ne,g),e(g,rt),e(g,ie),e(ie,lt),e(g,at),e(g,se),e(se,nt),e(g,it),e(g,fe),e(fe,st),e(g,ft),h(t,ke,f),h(t,k,f),e(k,ht),e(k,G),e(G,pt),e(k,dt),h(t,$e,f),h(t,M,f),e(M,ut),h(t,Pe,f),h(t,v,f),e(v,he),e(he,pe),e(pe,N),e(N,vt),e(v,ct),e(v,de),e(de,ue),e(ue,q),e(q,mt),e(v,gt),e(v,ve),e(ve,ce),e(ce,D),e(D,_t),e(v,wt),e(v,me),e(me,ge),e(ge,B),e(B,Et),e(v,bt),e(v,_e),e(_e,J),e(J,U),e(U,xt),e(J,yt),h(t,Ae,f),h(t,$,f),e($,Tt),e($,z),e(z,kt),e($,$t),h(t,Le,f),h(t,w,f),e(w,Pt),e(w,F),e(F,At),e(w,Lt),e(w,R),e(R,Ct),e(w,It),Ce=!0},p:Io,i(t){Ce||(qt(A.$$.fragment,t),qt(L.$$.fragment,t),qt(I.$$.fragment,t),Ce=!0)},o(t){Dt(A.$$.fragment,t),Dt(L.$$.fragment,t),Dt(I.$$.fragment,t),Ce=!1},d(t){o(_),t&&o(V),t&&o(c),Bt(A),t&&o(we),t&&o(x),Bt(L),t&&o(Ee),t&&o(T),t&&o(be),t&&o(j),t&&o(xe),Bt(I,t),t&&o(ye),t&&o(H),t&&o(Te),t&&o(u),t&&o(ke),t&&o(k),t&&o($e),t&&o(M),t&&o(Pe),t&&o(v),t&&o(Ae),t&&o($),t&&o(Le),t&&o(w)}}}const Go={local:"t5v11",sections:[{local:"overview",title:"Overview"}],title:"T5v1.1"};function No(De,_,V){let{fw:c}=_;return De.$$set=m=>{"fw"in m&&V(0,c=m.fw)},[c]}class Fo extends Po{constructor(_){super();Ao(this,_,No,So,Lo,{fw:0})}}export{Fo as default,Go as metadata};
