import{S as $o,i as Lo,s as Ao,e as r,k as p,w as St,t as n,M as Co,c as a,d as o,m as d,a as l,x as Gt,h as s,b as i,G as e,g as h,y as Ot,L as Io,q as Nt,o as qt,B as Dt,v as So}from"../../chunks/vendor-hf-doc-builder.js";import{I as Po}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as Go}from"../../chunks/CodeBlock-hf-doc-builder.js";function Oo(Ut){let w,_e,E,x,K,L,De,Q,Ue,we,b,y,W,A,Be,X,je,Ee,T,Re,C,Fe,Ve,be,F,Me,xe,I,ye,V,He,Te,u,Y,S,Je,G,ze,Ke,Qe,Z,ee,We,Xe,te,oe,Ye,Ze,re,ae,et,tt,le,v,ot,ne,rt,at,se,lt,nt,ie,st,it,ke,k,ft,O,ht,pt,Pe,M,dt,$e,c,fe,he,N,ut,ct,pe,de,q,vt,mt,ue,ce,D,gt,_t,ve,me,U,wt,Et,ge,H,B,bt,xt,Le,P,yt,J,Tt,kt,Ae,m,Pt,j,$t,Lt,R,At,Ct,Ce;return L=new Po({}),A=new Po({}),I=new Go({props:{code:`from transformers import T5ForConditionalGeneration

model = T5ForConditionalGeneration.from_pretrained("google/t5-v1_1-base")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> T5ForConditionalGeneration

<span class="hljs-meta">&gt;&gt;&gt; </span>model = T5ForConditionalGeneration.from_pretrained(<span class="hljs-string">&quot;google/t5-v1_1-base&quot;</span>)`}}),{c(){w=r("meta"),_e=p(),E=r("h1"),x=r("a"),K=r("span"),St(L.$$.fragment),De=p(),Q=r("span"),Ue=n("T5v1.1"),we=p(),b=r("h2"),y=r("a"),W=r("span"),St(A.$$.fragment),Be=p(),X=r("span"),je=n("Overview"),Ee=p(),T=r("p"),Re=n("T5v1.1 was released in the "),C=r("a"),Fe=n("google-research/text-to-text-transfer-transformer"),Ve=n(`
repository by Colin Raffel et al. It\u2019s an improved version of the original T5 model.`),be=p(),F=r("p"),Me=n("One can directly plug in the weights of T5v1.1 into a T5 model, like so:"),xe=p(),St(I.$$.fragment),ye=p(),V=r("p"),He=n("T5 Version 1.1 includes the following improvements compared to the original T5 model:"),Te=p(),u=r("ul"),Y=r("li"),S=r("p"),Je=n("GEGLU activation in the feed-forward hidden layer, rather than ReLU. See "),G=r("a"),ze=n("this paper"),Ke=n("."),Qe=p(),Z=r("li"),ee=r("p"),We=n("Dropout was turned off in pre-training (quality win). Dropout should be re-enabled during fine-tuning."),Xe=p(),te=r("li"),oe=r("p"),Ye=n("Pre-trained on C4 only without mixing in the downstream tasks."),Ze=p(),re=r("li"),ae=r("p"),et=n("No parameter sharing between the embedding and classifier layer."),tt=p(),le=r("li"),v=r("p"),ot=n("\u201Cxl\u201D and \u201Cxxl\u201D replace \u201C3B\u201D and \u201C11B\u201D. The model shapes are a bit different - larger "),ne=r("code"),rt=n("d_model"),at=n(` and smaller
`),se=r("code"),lt=n("num_heads"),nt=n(" and "),ie=r("code"),st=n("d_ff"),it=n("."),ke=p(),k=r("p"),ft=n("Note: T5 Version 1.1 was only pre-trained on "),O=r("a"),ht=n("C4"),pt=n(` excluding any supervised
training. Therefore, this model has to be fine-tuned before it is useable on a downstream task, unlike the original T5
model. Since t5v1.1 was pre-trained unsupervisedly, there\u2019s no real advantage to using a task prefix during single-task
fine-tuning. If you are doing multi-task fine-tuning, you should use a prefix.`),Pe=p(),M=r("p"),dt=n("Google has released the following variants:"),$e=p(),c=r("ul"),fe=r("li"),he=r("p"),N=r("a"),ut=n("google/t5-v1_1-small"),ct=p(),pe=r("li"),de=r("p"),q=r("a"),vt=n("google/t5-v1_1-base"),mt=p(),ue=r("li"),ce=r("p"),D=r("a"),gt=n("google/t5-v1_1-large"),_t=p(),ve=r("li"),me=r("p"),U=r("a"),wt=n("google/t5-v1_1-xl"),Et=p(),ge=r("li"),H=r("p"),B=r("a"),bt=n("google/t5-v1_1-xxl"),xt=n("."),Le=p(),P=r("p"),yt=n("One can refer to "),J=r("a"),Tt=n("T5\u2019s documentation page"),kt=n(" for all tips, code examples and notebooks."),Ae=p(),m=r("p"),Pt=n("This model was contributed by "),j=r("a"),$t=n("patrickvonplaten"),Lt=n(`. The original code can be
found `),R=r("a"),At=n("here"),Ct=n("."),this.h()},l(t){const f=Co('[data-svelte="svelte-1phssyn"]',document.head);w=a(f,"META",{name:!0,content:!0}),f.forEach(o),_e=d(t),E=a(t,"H1",{class:!0});var Ie=l(E);x=a(Ie,"A",{id:!0,class:!0,href:!0});var Bt=l(x);K=a(Bt,"SPAN",{});var jt=l(K);Gt(L.$$.fragment,jt),jt.forEach(o),Bt.forEach(o),De=d(Ie),Q=a(Ie,"SPAN",{});var Rt=l(Q);Ue=s(Rt,"T5v1.1"),Rt.forEach(o),Ie.forEach(o),we=d(t),b=a(t,"H2",{class:!0});var Se=l(b);y=a(Se,"A",{id:!0,class:!0,href:!0});var Ft=l(y);W=a(Ft,"SPAN",{});var Vt=l(W);Gt(A.$$.fragment,Vt),Vt.forEach(o),Ft.forEach(o),Be=d(Se),X=a(Se,"SPAN",{});var Mt=l(X);je=s(Mt,"Overview"),Mt.forEach(o),Se.forEach(o),Ee=d(t),T=a(t,"P",{});var Ge=l(T);Re=s(Ge,"T5v1.1 was released in the "),C=a(Ge,"A",{href:!0,rel:!0});var Ht=l(C);Fe=s(Ht,"google-research/text-to-text-transfer-transformer"),Ht.forEach(o),Ve=s(Ge,`
repository by Colin Raffel et al. It\u2019s an improved version of the original T5 model.`),Ge.forEach(o),be=d(t),F=a(t,"P",{});var Jt=l(F);Me=s(Jt,"One can directly plug in the weights of T5v1.1 into a T5 model, like so:"),Jt.forEach(o),xe=d(t),Gt(I.$$.fragment,t),ye=d(t),V=a(t,"P",{});var zt=l(V);He=s(zt,"T5 Version 1.1 includes the following improvements compared to the original T5 model:"),zt.forEach(o),Te=d(t),u=a(t,"UL",{});var g=l(u);Y=a(g,"LI",{});var Kt=l(Y);S=a(Kt,"P",{});var Oe=l(S);Je=s(Oe,"GEGLU activation in the feed-forward hidden layer, rather than ReLU. See "),G=a(Oe,"A",{href:!0,rel:!0});var Qt=l(G);ze=s(Qt,"this paper"),Qt.forEach(o),Ke=s(Oe,"."),Oe.forEach(o),Kt.forEach(o),Qe=d(g),Z=a(g,"LI",{});var Wt=l(Z);ee=a(Wt,"P",{});var Xt=l(ee);We=s(Xt,"Dropout was turned off in pre-training (quality win). Dropout should be re-enabled during fine-tuning."),Xt.forEach(o),Wt.forEach(o),Xe=d(g),te=a(g,"LI",{});var Yt=l(te);oe=a(Yt,"P",{});var Zt=l(oe);Ye=s(Zt,"Pre-trained on C4 only without mixing in the downstream tasks."),Zt.forEach(o),Yt.forEach(o),Ze=d(g),re=a(g,"LI",{});var eo=l(re);ae=a(eo,"P",{});var to=l(ae);et=s(to,"No parameter sharing between the embedding and classifier layer."),to.forEach(o),eo.forEach(o),tt=d(g),le=a(g,"LI",{});var oo=l(le);v=a(oo,"P",{});var $=l(v);ot=s($,"\u201Cxl\u201D and \u201Cxxl\u201D replace \u201C3B\u201D and \u201C11B\u201D. The model shapes are a bit different - larger "),ne=a($,"CODE",{});var ro=l(ne);rt=s(ro,"d_model"),ro.forEach(o),at=s($,` and smaller
`),se=a($,"CODE",{});var ao=l(se);lt=s(ao,"num_heads"),ao.forEach(o),nt=s($," and "),ie=a($,"CODE",{});var lo=l(ie);st=s(lo,"d_ff"),lo.forEach(o),it=s($,"."),$.forEach(o),oo.forEach(o),g.forEach(o),ke=d(t),k=a(t,"P",{});var Ne=l(k);ft=s(Ne,"Note: T5 Version 1.1 was only pre-trained on "),O=a(Ne,"A",{href:!0,rel:!0});var no=l(O);ht=s(no,"C4"),no.forEach(o),pt=s(Ne,` excluding any supervised
training. Therefore, this model has to be fine-tuned before it is useable on a downstream task, unlike the original T5
model. Since t5v1.1 was pre-trained unsupervisedly, there\u2019s no real advantage to using a task prefix during single-task
fine-tuning. If you are doing multi-task fine-tuning, you should use a prefix.`),Ne.forEach(o),Pe=d(t),M=a(t,"P",{});var so=l(M);dt=s(so,"Google has released the following variants:"),so.forEach(o),$e=d(t),c=a(t,"UL",{});var _=l(c);fe=a(_,"LI",{});var io=l(fe);he=a(io,"P",{});var fo=l(he);N=a(fo,"A",{href:!0,rel:!0});var ho=l(N);ut=s(ho,"google/t5-v1_1-small"),ho.forEach(o),fo.forEach(o),io.forEach(o),ct=d(_),pe=a(_,"LI",{});var po=l(pe);de=a(po,"P",{});var uo=l(de);q=a(uo,"A",{href:!0,rel:!0});var co=l(q);vt=s(co,"google/t5-v1_1-base"),co.forEach(o),uo.forEach(o),po.forEach(o),mt=d(_),ue=a(_,"LI",{});var vo=l(ue);ce=a(vo,"P",{});var mo=l(ce);D=a(mo,"A",{href:!0,rel:!0});var go=l(D);gt=s(go,"google/t5-v1_1-large"),go.forEach(o),mo.forEach(o),vo.forEach(o),_t=d(_),ve=a(_,"LI",{});var _o=l(ve);me=a(_o,"P",{});var wo=l(me);U=a(wo,"A",{href:!0,rel:!0});var Eo=l(U);wt=s(Eo,"google/t5-v1_1-xl"),Eo.forEach(o),wo.forEach(o),_o.forEach(o),Et=d(_),ge=a(_,"LI",{});var bo=l(ge);H=a(bo,"P",{});var It=l(H);B=a(It,"A",{href:!0,rel:!0});var xo=l(B);bt=s(xo,"google/t5-v1_1-xxl"),xo.forEach(o),xt=s(It,"."),It.forEach(o),bo.forEach(o),_.forEach(o),Le=d(t),P=a(t,"P",{});var qe=l(P);yt=s(qe,"One can refer to "),J=a(qe,"A",{href:!0});var yo=l(J);Tt=s(yo,"T5\u2019s documentation page"),yo.forEach(o),kt=s(qe," for all tips, code examples and notebooks."),qe.forEach(o),Ae=d(t),m=a(t,"P",{});var z=l(m);Pt=s(z,"This model was contributed by "),j=a(z,"A",{href:!0,rel:!0});var To=l(j);$t=s(To,"patrickvonplaten"),To.forEach(o),Lt=s(z,`. The original code can be
found `),R=a(z,"A",{href:!0,rel:!0});var ko=l(R);At=s(ko,"here"),ko.forEach(o),Ct=s(z,"."),z.forEach(o),this.h()},h(){i(w,"name","hf:doc:metadata"),i(w,"content",JSON.stringify(No)),i(x,"id","t5v11"),i(x,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(x,"href","#t5v11"),i(E,"class","relative group"),i(y,"id","overview"),i(y,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(y,"href","#overview"),i(b,"class","relative group"),i(C,"href","https://github.com/google-research/text-to-text-transfer-transformer/blob/main/released_checkpoints.md#t511"),i(C,"rel","nofollow"),i(G,"href","https://arxiv.org/abs/2002.05202"),i(G,"rel","nofollow"),i(O,"href","https://huggingface.co/datasets/c4"),i(O,"rel","nofollow"),i(N,"href","https://huggingface.co/google/t5-v1_1-small"),i(N,"rel","nofollow"),i(q,"href","https://huggingface.co/google/t5-v1_1-base"),i(q,"rel","nofollow"),i(D,"href","https://huggingface.co/google/t5-v1_1-large"),i(D,"rel","nofollow"),i(U,"href","https://huggingface.co/google/t5-v1_1-xl"),i(U,"rel","nofollow"),i(B,"href","https://huggingface.co/google/t5-v1_1-xxl"),i(B,"rel","nofollow"),i(J,"href","t5"),i(j,"href","https://huggingface.co/patrickvonplaten"),i(j,"rel","nofollow"),i(R,"href","https://github.com/google-research/text-to-text-transfer-transformer/blob/main/released_checkpoints.md#t511"),i(R,"rel","nofollow")},m(t,f){e(document.head,w),h(t,_e,f),h(t,E,f),e(E,x),e(x,K),Ot(L,K,null),e(E,De),e(E,Q),e(Q,Ue),h(t,we,f),h(t,b,f),e(b,y),e(y,W),Ot(A,W,null),e(b,Be),e(b,X),e(X,je),h(t,Ee,f),h(t,T,f),e(T,Re),e(T,C),e(C,Fe),e(T,Ve),h(t,be,f),h(t,F,f),e(F,Me),h(t,xe,f),Ot(I,t,f),h(t,ye,f),h(t,V,f),e(V,He),h(t,Te,f),h(t,u,f),e(u,Y),e(Y,S),e(S,Je),e(S,G),e(G,ze),e(S,Ke),e(u,Qe),e(u,Z),e(Z,ee),e(ee,We),e(u,Xe),e(u,te),e(te,oe),e(oe,Ye),e(u,Ze),e(u,re),e(re,ae),e(ae,et),e(u,tt),e(u,le),e(le,v),e(v,ot),e(v,ne),e(ne,rt),e(v,at),e(v,se),e(se,lt),e(v,nt),e(v,ie),e(ie,st),e(v,it),h(t,ke,f),h(t,k,f),e(k,ft),e(k,O),e(O,ht),e(k,pt),h(t,Pe,f),h(t,M,f),e(M,dt),h(t,$e,f),h(t,c,f),e(c,fe),e(fe,he),e(he,N),e(N,ut),e(c,ct),e(c,pe),e(pe,de),e(de,q),e(q,vt),e(c,mt),e(c,ue),e(ue,ce),e(ce,D),e(D,gt),e(c,_t),e(c,ve),e(ve,me),e(me,U),e(U,wt),e(c,Et),e(c,ge),e(ge,H),e(H,B),e(B,bt),e(H,xt),h(t,Le,f),h(t,P,f),e(P,yt),e(P,J),e(J,Tt),e(P,kt),h(t,Ae,f),h(t,m,f),e(m,Pt),e(m,j),e(j,$t),e(m,Lt),e(m,R),e(R,At),e(m,Ct),Ce=!0},p:Io,i(t){Ce||(Nt(L.$$.fragment,t),Nt(A.$$.fragment,t),Nt(I.$$.fragment,t),Ce=!0)},o(t){qt(L.$$.fragment,t),qt(A.$$.fragment,t),qt(I.$$.fragment,t),Ce=!1},d(t){o(w),t&&o(_e),t&&o(E),Dt(L),t&&o(we),t&&o(b),Dt(A),t&&o(Ee),t&&o(T),t&&o(be),t&&o(F),t&&o(xe),Dt(I,t),t&&o(ye),t&&o(V),t&&o(Te),t&&o(u),t&&o(ke),t&&o(k),t&&o(Pe),t&&o(M),t&&o($e),t&&o(c),t&&o(Le),t&&o(P),t&&o(Ae),t&&o(m)}}}const No={local:"t5v11",sections:[{local:"overview",title:"Overview"}],title:"T5v1.1"};function qo(Ut){return So(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class jo extends $o{constructor(w){super();Lo(this,w,qo,Oo,Ao,{})}}export{jo as default,No as metadata};
