import{S as rc,i as ac,s as sc,e as r,k as c,w as u,t,M as dc,c as a,d as o,m as l,a as s,x as g,h as n,b as i,F as e,g as w,y as _,q as v,o as b,B as T}from"../../chunks/vendor-4833417e.js";import{T as Vd}from"../../chunks/Tip-fffd6df1.js";import{D as B}from"../../chunks/Docstring-4f315ed9.js";import{C as yo}from"../../chunks/CodeBlock-6a3d1b46.js";import{I as Yt}from"../../chunks/IconCopyLink-4b81c553.js";import"../../chunks/CopyButton-dacfbfaf.js";function ic(Q){let p,P,m,M,F;return{c(){p=r("p"),P=t("Although the recipe for forward pass needs to be defined within this function, one should call the "),m=r("code"),M=t("Module"),F=t(`
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`)},l(y){p=a(y,"P",{});var j=s(p);P=n(j,"Although the recipe for forward pass needs to be defined within this function, one should call the "),m=a(j,"CODE",{});var C=s(m);M=n(C,"Module"),C.forEach(o),F=n(j,`
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`),j.forEach(o)},m(y,j){w(y,p,j),e(p,P),e(p,m),e(m,M),e(p,F)},d(y){y&&o(p)}}}function cc(Q){let p,P,m,M,F;return{c(){p=r("p"),P=t("Although the recipe for forward pass needs to be defined within this function, one should call the "),m=r("code"),M=t("Module"),F=t(`
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`)},l(y){p=a(y,"P",{});var j=s(p);P=n(j,"Although the recipe for forward pass needs to be defined within this function, one should call the "),m=a(j,"CODE",{});var C=s(m);M=n(C,"Module"),C.forEach(o),F=n(j,`
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`),j.forEach(o)},m(y,j){w(y,p,j),e(p,P),e(p,m),e(m,M),e(p,F)},d(y){y&&o(p)}}}function lc(Q){let p,P,m,M,F;return{c(){p=r("p"),P=t("Although the recipe for forward pass needs to be defined within this function, one should call the "),m=r("code"),M=t("Module"),F=t(`
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`)},l(y){p=a(y,"P",{});var j=s(p);P=n(j,"Although the recipe for forward pass needs to be defined within this function, one should call the "),m=a(j,"CODE",{});var C=s(m);M=n(C,"Module"),C.forEach(o),F=n(j,`
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`),j.forEach(o)},m(y,j){w(y,p,j),e(p,P),e(p,m),e(m,M),e(p,F)},d(y){y&&o(p)}}}function pc(Q){let p,P,m,M,F,y,j,C,Vn,Jt,f,$n,Eo,qn,zn,rt,Pn,Fn,ko,Cn,An,xo,Sn,In,jo,Ln,Nn,Do,On,Rn,at,Bn,Gn,Mo,Wn,Un,Vo,Hn,Zn,$o,Yn,Jn,qo,Kn,Qn,Kt,he,Xn,xe,er,or,Qt,G,tr,zo,nr,rr,Po,ar,sr,Xt,X,me,st,je,dr,dt,ir,en,z,De,cr,fe,Fo,lr,pr,Co,hr,mr,fr,ee,ur,Ao,gr,_r,So,vr,br,Tr,it,wr,yr,Me,Er,ue,Ve,kr,$e,xr,Io,jr,Dr,Mr,ge,qe,Vr,oe,$r,ct,qr,zr,lt,Pr,Fr,on,te,_e,pt,ze,Cr,ht,Ar,tn,E,Pe,Sr,ne,Ir,mt,Lr,Nr,ft,Or,Rr,Br,Fe,Gr,Ce,Wr,Ur,Hr,Ae,Zr,Se,Yr,Jr,Kr,ut,Qr,Xr,Ie,ea,Lo,oa,ta,na,Le,ra,Ne,aa,sa,da,W,No,ia,ca,gt,la,pa,_t,ha,ma,fa,A,Oe,ua,re,ga,Oo,_a,va,vt,ba,Ta,wa,ve,ya,bt,Ea,ka,Re,xa,S,Be,ja,Tt,Da,Ma,ae,Va,wt,$a,qa,yt,za,Pa,Fa,Et,Ca,Aa,Ge,nn,se,be,kt,We,Sa,xt,Ia,rn,k,Ue,La,de,Na,jt,Oa,Ra,Dt,Ba,Ga,Wa,He,Ua,Ze,Ha,Za,Ya,Ye,Ja,Je,Ka,Qa,Xa,Mt,es,os,Ke,ts,Ro,ns,rs,as,Qe,ss,Xe,ds,is,cs,U,Bo,ls,ps,Vt,hs,ms,$t,fs,us,gs,I,eo,_s,ie,vs,Go,bs,Ts,qt,ws,ys,Es,Te,ks,zt,xs,js,oo,Ds,O,to,Ms,Pt,Vs,$s,Ft,qs,zs,no,an,ce,we,Ct,ro,Ps,At,Fs,sn,x,ao,Cs,le,As,St,Ss,Is,It,Ls,Ns,Os,so,Rs,io,Bs,Gs,Ws,co,Us,lo,Hs,Zs,Ys,Lt,Js,Ks,po,Qs,Wo,Xs,ed,od,ho,td,mo,nd,rd,ad,H,Uo,sd,dd,Nt,id,cd,Ot,ld,pd,hd,L,fo,md,pe,fd,Ho,ud,gd,Rt,_d,vd,bd,ye,Td,Bt,wd,yd,uo,Ed,R,go,kd,Gt,xd,jd,Wt,Dd,Md,_o,dn;return y=new Yt({}),je=new Yt({}),De=new B({props:{name:"class transformers.VisionEncoderDecoderConfig",anchor:"transformers.VisionEncoderDecoderConfig",parameters:[{name:"**kwargs",val:""}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/models/vision_encoder_decoder/configuration_vision_encoder_decoder.py#L27",parametersDescription:[{anchor:"transformers.VisionEncoderDecoderConfig.kwargs",description:`<strong>kwargs</strong> (<em>optional</em>) &#x2014;
Dictionary of keyword arguments. Notably:</p>
<ul>
<li><strong>encoder</strong> (<a href="/docs/transformers/pr_15839/en/main_classes/configuration#transformers.PretrainedConfig">PretrainedConfig</a>, <em>optional</em>) &#x2014; An instance of a configuration object that defines
the encoder config.</li>
<li><strong>decoder</strong> (<a href="/docs/transformers/pr_15839/en/main_classes/configuration#transformers.PretrainedConfig">PretrainedConfig</a>, <em>optional</em>) &#x2014; An instance of a configuration object that defines
the decoder config.</li>
</ul>`,name:"kwargs"}]}}),Me=new yo({props:{code:`from transformers import BertConfig, ViTConfig, VisionEncoderDecoderConfig, VisionEncoderDecoderModel

# Initializing a ViT & BERT style configuration
config_encoder = ViTConfig()
config_decoder = BertConfig()

config = VisionEncoderDecoderConfig.from_encoder_decoder_configs(config_encoder, config_decoder)

# Initializing a ViTBert model from a ViT & bert-base-uncased style configurations
model = VisionEncoderDecoderModel(config=config)

# Accessing the model configuration
config_encoder = model.config.encoder
config_decoder = model.config.decoder
# set decoder config to causal lm
config_decoder.is_decoder = True
config_decoder.add_cross_attention = True

# Saving the model, including its configuration
model.save_pretrained("my-model")

# loading model and config from pretrained folder
encoder_decoder_config = VisionEncoderDecoderConfig.from_pretrained("my-model")
model = VisionEncoderDecoderModel.from_pretrained("my-model", config=encoder_decoder_config)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BertConfig, ViTConfig, VisionEncoderDecoderConfig, VisionEncoderDecoderModel

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a ViT &amp; BERT style configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>config_encoder = ViTConfig()
<span class="hljs-meta">&gt;&gt;&gt; </span>config_decoder = BertConfig()

<span class="hljs-meta">&gt;&gt;&gt; </span>config = VisionEncoderDecoderConfig.from_encoder_decoder_configs(config_encoder, config_decoder)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a ViTBert model from a ViT &amp; bert-base-uncased style configurations</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = VisionEncoderDecoderModel(config=config)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Accessing the model configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>config_encoder = model.config.encoder
<span class="hljs-meta">&gt;&gt;&gt; </span>config_decoder = model.config.decoder
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># set decoder config to causal lm</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>config_decoder.is_decoder = <span class="hljs-literal">True</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>config_decoder.add_cross_attention = <span class="hljs-literal">True</span>

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Saving the model, including its configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model.save_pretrained(<span class="hljs-string">&quot;my-model&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># loading model and config from pretrained folder</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>encoder_decoder_config = VisionEncoderDecoderConfig.from_pretrained(<span class="hljs-string">&quot;my-model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = VisionEncoderDecoderModel.from_pretrained(<span class="hljs-string">&quot;my-model&quot;</span>, config=encoder_decoder_config)`}}),Ve=new B({props:{name:"from_encoder_decoder_configs",anchor:"transformers.VisionEncoderDecoderConfig.from_encoder_decoder_configs",parameters:[{name:"encoder_config",val:": PretrainedConfig"},{name:"decoder_config",val:": PretrainedConfig"},{name:"**kwargs",val:""}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/models/vision_encoder_decoder/configuration_vision_encoder_decoder.py#L93",returnDescription:`
<p>An instance of a configuration object</p>
`,returnType:`
<p><a
  href="/docs/transformers/pr_15839/en/model_doc/vision-encoder-decoder#transformers.VisionEncoderDecoderConfig"
>VisionEncoderDecoderConfig</a></p>
`}}),qe=new B({props:{name:"to_dict",anchor:"transformers.VisionEncoderDecoderConfig.to_dict",parameters:[],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/models/vision_encoder_decoder/configuration_vision_encoder_decoder.py#L110",returnDescription:`
<p>Dictionary of all the attributes that make up this configuration instance,</p>
`,returnType:`
<p><code>Dict[str, any]</code></p>
`}}),ze=new Yt({}),Pe=new B({props:{name:"class transformers.VisionEncoderDecoderModel",anchor:"transformers.VisionEncoderDecoderModel",parameters:[{name:"config",val:": typing.Optional[transformers.configuration_utils.PretrainedConfig] = None"},{name:"encoder",val:": typing.Optional[transformers.modeling_utils.PreTrainedModel] = None"},{name:"decoder",val:": typing.Optional[transformers.modeling_utils.PreTrainedModel] = None"}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/models/vision_encoder_decoder/modeling_vision_encoder_decoder.py#L149",parametersDescription:[{anchor:"transformers.VisionEncoderDecoderModel.config",description:`<strong>config</strong> (<a href="/docs/transformers/pr_15839/en/model_doc/vision-encoder-decoder#transformers.VisionEncoderDecoderConfig">VisionEncoderDecoderConfig</a>) &#x2014; Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
configuration. Check out the <a href="/docs/transformers/pr_15839/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method to load the model weights.`,name:"config"}]}}),Oe=new B({props:{name:"forward",anchor:"transformers.VisionEncoderDecoderModel.forward",parameters:[{name:"pixel_values",val:" = None"},{name:"decoder_input_ids",val:" = None"},{name:"decoder_attention_mask",val:" = None"},{name:"encoder_outputs",val:" = None"},{name:"past_key_values",val:" = None"},{name:"decoder_inputs_embeds",val:" = None"},{name:"labels",val:" = None"},{name:"use_cache",val:" = None"},{name:"output_attentions",val:" = None"},{name:"output_hidden_states",val:" = None"},{name:"return_dict",val:" = None"},{name:"**kwargs",val:""}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/models/vision_encoder_decoder/modeling_vision_encoder_decoder.py#L401",parametersDescription:[{anchor:"transformers.VisionEncoderDecoderModel.forward.pixel_values",description:`<strong>pixel_values</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, num_channels, height, width)</code>) &#x2014;
Pixel values. Pixel values can be obtained using a feature extractor (e.g. if you use ViT as the encoder,
you should use <a href="/docs/transformers/pr_15839/en/model_doc/vit#transformers.ViTFeatureExtractor">ViTFeatureExtractor</a>). See <a href="/docs/transformers/pr_15839/en/model_doc/vit#transformers.ViTFeatureExtractor.__call__">ViTFeatureExtractor.<strong>call</strong>()</a> for details.`,name:"pixel_values"},{anchor:"transformers.VisionEncoderDecoderModel.forward.decoder_input_ids",description:`<strong>decoder_input_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, target_sequence_length)</code>, <em>optional</em>) &#x2014;
Indices of decoder input sequence tokens in the vocabulary.</p>
<p>Indices can be obtained using <a href="/docs/transformers/pr_15839/en/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a>. See <a href="/docs/transformers/pr_15839/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.encode">PreTrainedTokenizer.encode()</a> and
<a href="/docs/transformers/pr_15839/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details.</p>
<p><a href="../glossary#input-ids">What are input IDs?</a></p>
<p>If <code>past_key_values</code> is used, optionally only the last <code>decoder_input_ids</code> have to be input (see
<code>past_key_values</code>).</p>
<p>For training, <code>decoder_input_ids</code> are automatically created by the model by shifting the <code>labels</code> to the
right, replacing -100 by the <code>pad_token_id</code> and prepending them with the <code>decoder_start_token_id</code>.`,name:"decoder_input_ids"},{anchor:"transformers.VisionEncoderDecoderModel.forward.decoder_attention_mask",description:`<strong>decoder_attention_mask</strong> (<code>torch.BoolTensor</code> of shape <code>(batch_size, target_sequence_length)</code>, <em>optional</em>) &#x2014;
Default behavior: generate a tensor that ignores pad tokens in <code>decoder_input_ids</code>. Causal mask will also
be used by default.`,name:"decoder_attention_mask"},{anchor:"transformers.VisionEncoderDecoderModel.forward.encoder_outputs",description:`<strong>encoder_outputs</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>) &#x2014;
This tuple must consist of (<code>last_hidden_state</code>, <em>optional</em>: <code>hidden_states</code>, <em>optional</em>: <code>attentions</code>)
<code>last_hidden_state</code> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>) is a tensor
of hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the
decoder.`,name:"encoder_outputs"},{anchor:"transformers.VisionEncoderDecoderModel.forward.past_key_values",description:`<strong>past_key_values</strong> (<code>tuple(tuple(torch.FloatTensor))</code> of length <code>config.n_layers</code> with each tuple having 4 tensors of shape <code>(batch_size, num_heads, sequence_length - 1, embed_size_per_head)</code>) &#x2014;
Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.</p>
<p>If <code>past_key_values</code> are used, the user can optionally input only the last <code>decoder_input_ids</code> (those that
don&#x2019;t have their past key value states given to this model) of shape <code>(batch_size, 1)</code> instead of all
<code>decoder_input_ids</code> of shape <code>(batch_size, sequence_length)</code>.`,name:"past_key_values"},{anchor:"transformers.VisionEncoderDecoderModel.forward.decoder_inputs_embeds",description:`<strong>decoder_inputs_embeds</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, target_sequence_length, hidden_size)</code>, <em>optional</em>) &#x2014;
Optionally, instead of passing <code>decoder_input_ids</code> you can choose to directly pass an embedded
representation. This is useful if you want more control over how to convert <code>decoder_input_ids</code> indices
into associated vectors than the model&#x2019;s internal embedding lookup matrix.`,name:"decoder_inputs_embeds"},{anchor:"transformers.VisionEncoderDecoderModel.forward.labels",description:`<strong>labels</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Labels for computing the masked language modeling loss for the decoder. Indices should be in <code>[-100, 0, ..., config.vocab_size]</code> (see <code>input_ids</code> docstring) Tokens with indices set to <code>-100</code> are ignored
(masked), the loss is only computed for the tokens with labels in <code>[0, ..., config.vocab_size]</code>`,name:"labels"},{anchor:"transformers.VisionEncoderDecoderModel.forward.use_cache",description:`<strong>use_cache</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
If set to <code>True</code>, <code>past_key_values</code> key value states are returned and can be used to speed up decoding (see
<code>past_key_values</code>).`,name:"use_cache"},{anchor:"transformers.VisionEncoderDecoderModel.forward.output_attentions",description:`<strong>output_attentions</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the attentions tensors of all attention layers. See <code>attentions</code> under returned
tensors for more detail.`,name:"output_attentions"},{anchor:"transformers.VisionEncoderDecoderModel.forward.output_hidden_states",description:`<strong>output_hidden_states</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the hidden states of all layers. See <code>hidden_states</code> under returned tensors for
more detail.`,name:"output_hidden_states"},{anchor:"transformers.VisionEncoderDecoderModel.forward.return_dict",description:`<strong>return_dict</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
If set to <code>True</code>, the model will return a <code>Seq2SeqLMOutput</code> instead of a plain tuple.
kwargs &#x2014; (<em>optional</em>) Remaining dictionary of keyword arguments. Keyword arguments come in two flavors:</p>
<ul>
<li>Without a prefix which will be input as <code>**encoder_kwargs</code> for the encoder forward function.</li>
<li>With a <em>decoder_</em> prefix which will be input as <code>**decoder_kwargs</code> for the decoder forward function.</li>
</ul>`,name:"return_dict"}],returnDescription:`
<p>A <a
  href="/docs/transformers/pr_15839/en/main_classes/output#transformers.modeling_outputs.Seq2SeqLMOutput"
>transformers.modeling_outputs.Seq2SeqLMOutput</a> or a tuple of
<code>torch.FloatTensor</code> (if <code>return_dict=False</code> is passed or when <code>config.return_dict=False</code>) comprising various
elements depending on the configuration (<a
  href="/docs/transformers/pr_15839/en/model_doc/vision-encoder-decoder#transformers.VisionEncoderDecoderConfig"
>VisionEncoderDecoderConfig</a>) and inputs.</p>
<ul>
<li>
<p><strong>loss</strong> (<code>torch.FloatTensor</code> of shape <code>(1,)</code>, <em>optional</em>, returned when <code>labels</code> is provided) \u2014 Language modeling loss.</p>
</li>
<li>
<p><strong>logits</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, config.vocab_size)</code>) \u2014 Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).</p>
</li>
<li>
<p><strong>past_key_values</strong> (<code>tuple(tuple(torch.FloatTensor))</code>, <em>optional</em>, returned when <code>use_cache=True</code> is passed or when <code>config.use_cache=True</code>) \u2014 Tuple of <code>tuple(torch.FloatTensor)</code> of length <code>config.n_layers</code>, with each tuple having 2 tensors of shape
<code>(batch_size, num_heads, sequence_length, embed_size_per_head)</code>) and 2 additional tensors of shape
<code>(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)</code>.</p>
<p>Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
blocks) that can be used (see <code>past_key_values</code> input) to speed up sequential decoding.</p>
</li>
<li>
<p><strong>decoder_hidden_states</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) \u2014 Tuple of <code>torch.FloatTensor</code> (one for the output of the embeddings + one for the output of each layer) of
shape <code>(batch_size, sequence_length, hidden_size)</code>.</p>
<p>Hidden-states of the decoder at the output of each layer plus the initial embedding outputs.</p>
</li>
<li>
<p><strong>decoder_attentions</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) \u2014 Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p>
<p>Attentions weights of the decoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.</p>
</li>
<li>
<p><strong>cross_attentions</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) \u2014 Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p>
<p>Attentions weights of the decoder\u2019s cross-attention layer, after the attention softmax, used to compute the
weighted average in the cross-attention heads.</p>
</li>
<li>
<p><strong>encoder_last_hidden_state</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>, <em>optional</em>) \u2014 Sequence of hidden-states at the output of the last layer of the encoder of the model.</p>
</li>
<li>
<p><strong>encoder_hidden_states</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) \u2014 Tuple of <code>torch.FloatTensor</code> (one for the output of the embeddings + one for the output of each layer) of
shape <code>(batch_size, sequence_length, hidden_size)</code>.</p>
<p>Hidden-states of the encoder at the output of each layer plus the initial embedding outputs.</p>
</li>
<li>
<p><strong>encoder_attentions</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) \u2014 Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p>
<p>Attentions weights of the encoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.</p>
</li>
</ul>
`,returnType:`
<p><a
  href="/docs/transformers/pr_15839/en/main_classes/output#transformers.modeling_outputs.Seq2SeqLMOutput"
>transformers.modeling_outputs.Seq2SeqLMOutput</a> or <code>tuple(torch.FloatTensor)</code></p>
`}}),ve=new Vd({props:{$$slots:{default:[ic]},$$scope:{ctx:Q}}}),Re=new yo({props:{code:`from transformers import TrOCRProcessor, VisionEncoderDecoderModel
import requests
from PIL import Image
import torch

processor = TrOCRProcessor.from_pretrained("microsoft/trocr-base-handwritten")
model = VisionEncoderDecoderModel.from_pretrained("microsoft/trocr-base-handwritten")

# load image from the IAM dataset
url = "https://fki.tic.heia-fr.ch/static/img/a01-122-02.jpg"
image = Image.open(requests.get(url, stream=True).raw).convert("RGB")

# training
model.config.decoder_start_token_id = processor.tokenizer.cls_token_id
model.config.pad_token_id = processor.tokenizer.pad_token_id
model.config.vocab_size = model.config.decoder.vocab_size

pixel_values = processor(image, return_tensors="pt").pixel_values
text = "hello world"
labels = processor.tokenizer(text, return_tensors="pt").input_ids
outputs = model(pixel_values=pixel_values, labels=labels)
loss = outputs.loss

# inference (generation)
generated_ids = model.generate(pixel_values)
generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TrOCRProcessor, VisionEncoderDecoderModel
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> requests
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch

<span class="hljs-meta">&gt;&gt;&gt; </span>processor = TrOCRProcessor.from_pretrained(<span class="hljs-string">&quot;microsoft/trocr-base-handwritten&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = VisionEncoderDecoderModel.from_pretrained(<span class="hljs-string">&quot;microsoft/trocr-base-handwritten&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># load image from the IAM dataset</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>url = <span class="hljs-string">&quot;https://fki.tic.heia-fr.ch/static/img/a01-122-02.jpg&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>image = Image.<span class="hljs-built_in">open</span>(requests.get(url, stream=<span class="hljs-literal">True</span>).raw).convert(<span class="hljs-string">&quot;RGB&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># training</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model.config.decoder_start_token_id = processor.tokenizer.cls_token_id
<span class="hljs-meta">&gt;&gt;&gt; </span>model.config.pad_token_id = processor.tokenizer.pad_token_id
<span class="hljs-meta">&gt;&gt;&gt; </span>model.config.vocab_size = model.config.decoder.vocab_size

<span class="hljs-meta">&gt;&gt;&gt; </span>pixel_values = processor(image, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).pixel_values
<span class="hljs-meta">&gt;&gt;&gt; </span>text = <span class="hljs-string">&quot;hello world&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>labels = processor.tokenizer(text, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).input_ids
<span class="hljs-meta">&gt;&gt;&gt; </span>outputs = model(pixel_values=pixel_values, labels=labels)
<span class="hljs-meta">&gt;&gt;&gt; </span>loss = outputs.loss

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># inference (generation)</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>generated_ids = model.generate(pixel_values)
<span class="hljs-meta">&gt;&gt;&gt; </span>generated_text = processor.batch_decode(generated_ids, skip_special_tokens=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>]`}}),Be=new B({props:{name:"from_encoder_decoder_pretrained",anchor:"transformers.VisionEncoderDecoderModel.from_encoder_decoder_pretrained",parameters:[{name:"encoder_pretrained_model_name_or_path",val:": str = None"},{name:"decoder_pretrained_model_name_or_path",val:": str = None"},{name:"*model_args",val:""},{name:"**kwargs",val:""}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/models/vision_encoder_decoder/modeling_vision_encoder_decoder.py#L246",parametersDescription:[{anchor:"transformers.VisionEncoderDecoderModel.from_encoder_decoder_pretrained.encoder_pretrained_model_name_or_path",description:`<strong>encoder_pretrained_model_name_or_path</strong> (<code>str</code>, <em>optional</em>) &#x2014;
Information necessary to initiate the image encoder. Can be either:</p>
<ul>
<li>A string, the <em>model id</em> of a pretrained model hosted inside a model repo on huggingface.co. An
example is <code>google/vit-base-patch16-224-in21k</code>.</li>
<li>A path to a <em>directory</em> containing model weights saved using
<a href="/docs/transformers/pr_15839/en/main_classes/model#transformers.PreTrainedModel.save_pretrained">save_pretrained()</a>, e.g., <code>./my_model_directory/</code>.</li>
<li>A path or url to a <em>tensorflow index checkpoint file</em> (e.g, <code>./tf_model/model.ckpt.index</code>). In
this case, <code>from_tf</code> should be set to <code>True</code> and a configuration object should be provided as
<code>config</code> argument. This loading path is slower than converting the TensorFlow checkpoint in a
PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.</li>
</ul>`,name:"encoder_pretrained_model_name_or_path"},{anchor:"transformers.VisionEncoderDecoderModel.from_encoder_decoder_pretrained.decoder_pretrained_model_name_or_path",description:`<strong>decoder_pretrained_model_name_or_path</strong> (<code>str</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Information necessary to initiate the text decoder. Can be either:</p>
<ul>
<li>A string, the <em>model id</em> of a pretrained model hosted inside a model repo on huggingface.co.
Valid model ids can be located at the root-level, like <code>bert-base-uncased</code>, or namespaced under a
user or organization name, like <code>dbmdz/bert-base-german-cased</code>.</li>
<li>A path to a <em>directory</em> containing model weights saved using
<a href="/docs/transformers/pr_15839/en/main_classes/model#transformers.PreTrainedModel.save_pretrained">save_pretrained()</a>, e.g., <code>./my_model_directory/</code>.</li>
<li>A path or url to a <em>tensorflow index checkpoint file</em> (e.g, <code>./tf_model/model.ckpt.index</code>). In
this case, <code>from_tf</code> should be set to <code>True</code> and a configuration object should be provided as
<code>config</code> argument. This loading path is slower than converting the TensorFlow checkpoint in a
PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.</li>
</ul>`,name:"decoder_pretrained_model_name_or_path"},{anchor:"transformers.VisionEncoderDecoderModel.from_encoder_decoder_pretrained.model_args",description:`<strong>model_args</strong> (remaining positional arguments, <em>optional</em>) &#x2014;
All remaning positional arguments will be passed to the underlying model&#x2019;s <code>__init__</code> method.`,name:"model_args"},{anchor:"transformers.VisionEncoderDecoderModel.from_encoder_decoder_pretrained.kwargs",description:`<strong>kwargs</strong> (remaining dictionary of keyword arguments, <em>optional</em>) &#x2014;
Can be used to update the configuration object (after it being loaded) and initiate the model (e.g.,
<code>output_attentions=True</code>).</p>
<ul>
<li>To update the encoder configuration, use the prefix <em>encoder_</em> for each configuration parameter.</li>
<li>To update the decoder configuration, use the prefix <em>decoder_</em> for each configuration parameter.</li>
<li>To update the parent model configuration, do not use a prefix for each configuration parameter.</li>
</ul>
<p>Behaves differently depending on whether a <code>config</code> is provided or automatically loaded.`,name:"kwargs"}]}}),Ge=new yo({props:{code:`from transformers import VisionEncoderDecoderModel

# initialize a vit-bert from a pretrained ViT and a pretrained BERT model. Note that the cross-attention layers will be randomly initialized
model = VisionEncoderDecoderModel.from_encoder_decoder_pretrained(
    "google/vit-base-patch16-224-in21k", "bert-base-uncased"
)
# saving model after fine-tuning
model.save_pretrained("./vit-bert")
# load fine-tuned model
model = VisionEncoderDecoderModel.from_pretrained("./vit-bert")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> VisionEncoderDecoderModel

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># initialize a vit-bert from a pretrained ViT and a pretrained BERT model. Note that the cross-attention layers will be randomly initialized</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = VisionEncoderDecoderModel.from_encoder_decoder_pretrained(
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;google/vit-base-patch16-224-in21k&quot;</span>, <span class="hljs-string">&quot;bert-base-uncased&quot;</span>
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># saving model after fine-tuning</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model.save_pretrained(<span class="hljs-string">&quot;./vit-bert&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># load fine-tuned model</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = VisionEncoderDecoderModel.from_pretrained(<span class="hljs-string">&quot;./vit-bert&quot;</span>)`}}),We=new Yt({}),Ue=new B({props:{name:"class transformers.TFVisionEncoderDecoderModel",anchor:"transformers.TFVisionEncoderDecoderModel",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/models/vision_encoder_decoder/modeling_tf_vision_encoder_decoder.py#L176",parametersDescription:[{anchor:"transformers.TFVisionEncoderDecoderModel.config",description:`<strong>config</strong> (<a href="/docs/transformers/pr_15839/en/model_doc/vision-encoder-decoder#transformers.VisionEncoderDecoderConfig">VisionEncoderDecoderConfig</a>) &#x2014; Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
configuration. Check out the <a href="/docs/transformers/pr_15839/en/main_classes/model#transformers.TFPreTrainedModel.from_pretrained">from_pretrained()</a> method to load the model weights.`,name:"config"}]}}),eo=new B({props:{name:"call",anchor:"transformers.TFVisionEncoderDecoderModel.call",parameters:[{name:"pixel_values",val:" = None"},{name:"decoder_input_ids",val:" = None"},{name:"decoder_attention_mask",val:" = None"},{name:"encoder_outputs",val:" = None"},{name:"past_key_values",val:" = None"},{name:"decoder_inputs_embeds",val:" = None"},{name:"labels",val:" = None"},{name:"use_cache",val:" = None"},{name:"output_attentions",val:" = None"},{name:"output_hidden_states",val:" = None"},{name:"return_dict",val:" = None"},{name:"training",val:" = False"},{name:"**kwargs",val:""}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/models/vision_encoder_decoder/modeling_tf_vision_encoder_decoder.py#L513",parametersDescription:[{anchor:"transformers.TFVisionEncoderDecoderModel.call.pixel_values",description:`<strong>pixel_values</strong> (<code>np.ndarray</code>, <code>tf.Tensor</code>, <code>List[tf.Tensor]</code> \`<code>Dict[str, tf.Tensor]</code> or <code>Dict[str, np.ndarray]</code> and each example must have the shape <code>(batch_size, num_channels, height, width)</code>) &#x2014;
Pixel values. Pixel values can be obtained using the vision&#x2019;s model&#x2019;s feature extractor. For example, using
<a href="/docs/transformers/pr_15839/en/model_doc/vit#transformers.ViTFeatureExtractor">ViTFeatureExtractor</a>. See <a href="/docs/transformers/pr_15839/en/model_doc/vit#transformers.ViTFeatureExtractor.__call__">ViTFeatureExtractor.<strong>call</strong>()</a> for details.`,name:"pixel_values"},{anchor:"transformers.TFVisionEncoderDecoderModel.call.decoder_input_ids",description:`<strong>decoder_input_ids</strong> (<code>np.ndarray</code> or <code>tf.Tensor</code> of shape <code>(batch_size, target_sequence_length)</code>, <em>optional</em>) &#x2014;
Indices of decoder input sequence tokens in the vocabulary.</p>
<p>Indices can be obtained using <a href="/docs/transformers/pr_15839/en/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a>. See <a href="/docs/transformers/pr_15839/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.encode">PreTrainedTokenizer.encode()</a> and
<a href="/docs/transformers/pr_15839/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details.</p>
<p><a href="../glossary#input-ids">What are input IDs?</a></p>
<p>If <code>past_key_values</code> is used, optionally only the last <code>decoder_input_ids</code> have to be input (see
<code>past_key_values</code>).</p>
<p>Provide for sequence to sequence training to the decoder. Indices can be obtained using
<a href="/docs/transformers/pr_15839/en/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a>. See <a href="/docs/transformers/pr_15839/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.encode">PreTrainedTokenizer.encode()</a> and <a href="/docs/transformers/pr_15839/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for
details.`,name:"decoder_input_ids"},{anchor:"transformers.TFVisionEncoderDecoderModel.call.decoder_attention_mask",description:`<strong>decoder_attention_mask</strong> (<code>np.ndarray</code> or <code>tf.Tensor</code> of shape <code>(batch_size, target_sequence_length)</code>, <em>optional</em>) &#x2014;
Default behavior: generate a tensor that ignores pad tokens in <code>decoder_input_ids</code>. Causal mask will also
be used by default.`,name:"decoder_attention_mask"},{anchor:"transformers.TFVisionEncoderDecoderModel.call.encoder_outputs",description:`<strong>encoder_outputs</strong> (<code>tuple(tuple(tf.Tensor)</code>, <em>optional</em>) &#x2014;
This tuple must consist of (<code>last_hidden_state</code>, <em>optional</em>: <code>hidden_states</code>, <em>optional</em>: <code>attentions</code>)
<code>last_hidden_state</code> (<code>tf.Tensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>) is a tensor of hidden-states at the output
of the last layer of the encoder. Used in the cross-attention of the decoder.`,name:"encoder_outputs"},{anchor:"transformers.TFVisionEncoderDecoderModel.call.past_key_values",description:`<strong>past_key_values</strong> (<code>tuple(tuple(tf.Tensor))</code> of length <code>config.n_layers</code> with each tuple having 4 tensors of shape <code>(batch_size, num_heads, sequence_length - 1, embed_size_per_head)</code>) &#x2014;
Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.</p>
<p>If <code>past_key_values</code> are used, the user can optionally input only the last <code>decoder_input_ids</code> (those that
don&#x2019;t have their past key value states given to this model) of shape <code>(batch_size, 1)</code> instead of all
<code>decoder_input_ids</code> of shape <code>(batch_size, sequence_length)</code>.`,name:"past_key_values"},{anchor:"transformers.TFVisionEncoderDecoderModel.call.decoder_inputs_embeds",description:`<strong>decoder_inputs_embeds</strong> (<code>np.ndarray</code> or <code>tf.Tensor</code> of shape <code>(batch_size, target_sequence_length, hidden_size)</code>, <em>optional</em>) &#x2014;
Optionally, instead of passing <code>decoder_input_ids</code> you can choose to directly pass an embedded
representation. This is useful if you want more control over how to convert <code>decoder_input_ids</code> indices
into associated vectors than the model&#x2019;s internal embedding lookup matrix.`,name:"decoder_inputs_embeds"},{anchor:"transformers.TFVisionEncoderDecoderModel.call.labels",description:`<strong>labels</strong> (<code>np.ndarray</code> or <code>tf.Tensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Labels for computing the masked language modeling loss for the decoder. Indices should be in <code>[-100, 0, ..., config.vocab_size]</code> (see <code>input_ids</code> docstring) Tokens with indices set to <code>-100</code> are ignored
(masked), the loss is only computed for the tokens with labels in <code>[0, ..., config.vocab_size]</code>`,name:"labels"},{anchor:"transformers.TFVisionEncoderDecoderModel.call.use_cache",description:`<strong>use_cache</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
If set to <code>True</code>, <code>past_key_values</code> key value states are returned and can be used to speed up decoding (see
<code>past_key_values</code>).`,name:"use_cache"},{anchor:"transformers.TFVisionEncoderDecoderModel.call.output_attentions",description:`<strong>output_attentions</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the attentions tensors of all attention layers. See <code>attentions</code> under returned
tensors for more detail.`,name:"output_attentions"},{anchor:"transformers.TFVisionEncoderDecoderModel.call.output_hidden_states",description:`<strong>output_hidden_states</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the hidden states of all layers. See <code>hidden_states</code> under returned tensors for
more detail.`,name:"output_hidden_states"},{anchor:"transformers.TFVisionEncoderDecoderModel.call.return_dict",description:`<strong>return_dict</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
If set to <code>True</code>, the model will return a <code>Seq2SeqLMOutput</code> instead of a plain tuple.`,name:"return_dict"},{anchor:"transformers.TFVisionEncoderDecoderModel.call.training",description:`<strong>training</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
kwargs &#x2014; (<em>optional</em>) Remaining dictionary of keyword arguments. Keyword arguments come in two flavors:</p>
<ul>
<li>Without a prefix which will be input as <code>**encoder_kwargs</code> for the encoder forward function.</li>
<li>With a <em>decoder_</em> prefix which will be input as <code>**decoder_kwargs</code> for the decoder forward function.</li>
</ul>`,name:"training"}],returnDescription:`
<p>A <a
  href="/docs/transformers/pr_15839/en/main_classes/output#transformers.modeling_tf_outputs.TFSeq2SeqLMOutput"
>transformers.modeling_tf_outputs.TFSeq2SeqLMOutput</a> or a tuple of <code>tf.Tensor</code> (if
<code>return_dict=False</code> is passed or when <code>config.return_dict=False</code>) comprising various elements depending on the
configuration (<a
  href="/docs/transformers/pr_15839/en/model_doc/vision-encoder-decoder#transformers.VisionEncoderDecoderConfig"
>VisionEncoderDecoderConfig</a>) and inputs.</p>
<ul>
<li>
<p><strong>loss</strong> (<code>tf.Tensor</code> of shape <code>(n,)</code>, <em>optional</em>, where n is the number of non-masked labels, returned when <code>labels</code> is provided) \u2014 Language modeling loss.</p>
</li>
<li>
<p><strong>logits</strong> (<code>tf.Tensor</code> of shape <code>(batch_size, sequence_length, config.vocab_size)</code>) \u2014 Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).</p>
</li>
<li>
<p><strong>past_key_values</strong> (<code>List[tf.Tensor]</code>, <em>optional</em>, returned when <code>use_cache=True</code> is passed or when <code>config.use_cache=True</code>) \u2014 List of <code>tf.Tensor</code> of length <code>config.n_layers</code>, with each tensor of shape <code>(2, batch_size, num_heads, sequence_length, embed_size_per_head)</code>).</p>
<p>Contains pre-computed hidden-states (key and values in the attention blocks) of the decoder that can be
used (see <code>past_key_values</code> input) to speed up sequential decoding.</p>
</li>
<li>
<p><strong>decoder_hidden_states</strong> (<code>tuple(tf.Tensor)</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) \u2014 Tuple of <code>tf.Tensor</code> (one for the output of the embeddings + one for the output of each layer) of shape
<code>(batch_size, sequence_length, hidden_size)</code>.</p>
<p>Hidden-states of the decoder at the output of each layer plus the initial embedding outputs.</p>
</li>
<li>
<p><strong>decoder_attentions</strong> (<code>tuple(tf.Tensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) \u2014 Tuple of <code>tf.Tensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p>
<p>Attentions weights of the decoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.</p>
</li>
<li>
<p><strong>cross_attentions</strong> (<code>tuple(tf.Tensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) \u2014 Tuple of <code>tf.Tensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p>
<p>Attentions weights of the decoder\u2019s cross-attention layer, after the attention softmax, used to compute the
weighted average in the cross-attention heads.</p>
</li>
<li>
<p><strong>encoder_last_hidden_state</strong> (<code>tf.Tensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>, <em>optional</em>) \u2014 Sequence of hidden-states at the output of the last layer of the encoder of the model.</p>
</li>
<li>
<p><strong>encoder_hidden_states</strong> (<code>tuple(tf.Tensor)</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) \u2014 Tuple of <code>tf.Tensor</code> (one for the output of the embeddings + one for the output of each layer) of shape
<code>(batch_size, sequence_length, hidden_size)</code>.</p>
<p>Hidden-states of the encoder at the output of each layer plus the initial embedding outputs.</p>
</li>
<li>
<p><strong>encoder_attentions</strong> (<code>tuple(tf.Tensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) \u2014 Tuple of <code>tf.Tensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p>
<p>Attentions weights of the encoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.</p>
</li>
</ul>
`,returnType:`
<p><a
  href="/docs/transformers/pr_15839/en/main_classes/output#transformers.modeling_tf_outputs.TFSeq2SeqLMOutput"
>transformers.modeling_tf_outputs.TFSeq2SeqLMOutput</a> or <code>tuple(tf.Tensor)</code></p>
`}}),Te=new Vd({props:{$$slots:{default:[cc]},$$scope:{ctx:Q}}}),oo=new yo({props:{code:`from transformers import AutoFeatureExtractor, AutoTokenizer, TFVisionEncoderDecoderModel
from PIL import Image
import requests

feature_extractor = AutoFeatureExtractor.from_pretrained("google/vit-base-patch16-224-in21k")
decoder_tokenizer = AutoTokenizer.from_pretrained("gpt2")

# initialize a bert2gpt2 from a pretrained BERT and GPT2 models. Note that the cross-attention layers will be randomly initialized
model = TFVisionEncoderDecoderModel.from_encoder_decoder_pretrained(
    "google/vit-base-patch16-224-in21k", "gpt2"
)

url = "http://images.cocodataset.org/val2017/000000039769.jpg"
img = Image.open(requests.get(url, stream=True).raw)

# forward
pixel_values = feature_extractor(images=img, return_tensors="tf").pixel_values  # Batch size 1
decoder_input_ids = decoder_tokenizer("Linda Davis", return_tensors="tf").input_ids  # Batch size 1
outputs = model(pixel_values=pixel_values, decoder_input_ids=decoder_input_ids)

# training
outputs = model(pixel_values=pixel_values, decoder_input_ids=decoder_input_ids, labels=decoder_input_ids)
loss, logits = outputs.loss, outputs.logits

# save and load from pretrained
model.save_pretrained("vit-gpt2")
model = TFVisionEncoderDecoderModel.from_pretrained("vit-gpt2")

# generation
generated = model.generate(pixel_values, decoder_start_token_id=model.config.decoder.bos_token_id)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoFeatureExtractor, AutoTokenizer, TFVisionEncoderDecoderModel
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> requests

<span class="hljs-meta">&gt;&gt;&gt; </span>feature_extractor = AutoFeatureExtractor.from_pretrained(<span class="hljs-string">&quot;google/vit-base-patch16-224-in21k&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>decoder_tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;gpt2&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># initialize a bert2gpt2 from a pretrained BERT and GPT2 models. Note that the cross-attention layers will be randomly initialized</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = TFVisionEncoderDecoderModel.from_encoder_decoder_pretrained(
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;google/vit-base-patch16-224-in21k&quot;</span>, <span class="hljs-string">&quot;gpt2&quot;</span>
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>url = <span class="hljs-string">&quot;http://images.cocodataset.org/val2017/000000039769.jpg&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>img = Image.<span class="hljs-built_in">open</span>(requests.get(url, stream=<span class="hljs-literal">True</span>).raw)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># forward</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>pixel_values = feature_extractor(images=img, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>).pixel_values  <span class="hljs-comment"># Batch size 1</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>decoder_input_ids = decoder_tokenizer(<span class="hljs-string">&quot;Linda Davis&quot;</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>).input_ids  <span class="hljs-comment"># Batch size 1</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>outputs = model(pixel_values=pixel_values, decoder_input_ids=decoder_input_ids)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># training</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>outputs = model(pixel_values=pixel_values, decoder_input_ids=decoder_input_ids, labels=decoder_input_ids)
<span class="hljs-meta">&gt;&gt;&gt; </span>loss, logits = outputs.loss, outputs.logits

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># save and load from pretrained</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model.save_pretrained(<span class="hljs-string">&quot;vit-gpt2&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = TFVisionEncoderDecoderModel.from_pretrained(<span class="hljs-string">&quot;vit-gpt2&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># generation</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>generated = model.generate(pixel_values, decoder_start_token_id=model.config.decoder.bos_token_id)`}}),to=new B({props:{name:"from_encoder_decoder_pretrained",anchor:"transformers.TFVisionEncoderDecoderModel.from_encoder_decoder_pretrained",parameters:[{name:"encoder_pretrained_model_name_or_path",val:": str = None"},{name:"decoder_pretrained_model_name_or_path",val:": str = None"},{name:"*model_args",val:""},{name:"**kwargs",val:""}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/models/vision_encoder_decoder/modeling_tf_vision_encoder_decoder.py#L348",parametersDescription:[{anchor:"transformers.TFVisionEncoderDecoderModel.from_encoder_decoder_pretrained.encoder_pretrained_model_name_or_path",description:`<strong>encoder_pretrained_model_name_or_path</strong> (<code>str</code>, <em>optional</em>) &#x2014;
Information necessary to initiate the encoder. Can be either:</p>
<ul>
<li>A string, the <em>model id</em> of a pretrained model hosted inside a model repo on huggingface.co. An
example is <code>google/vit-base-patch16-224-in21k</code>.</li>
<li>A path to a <em>directory</em> containing model weights saved using
<a href="/docs/transformers/pr_15839/en/main_classes/model#transformers.TFPreTrainedModel.save_pretrained">save_pretrained()</a>, e.g., <code>./my_model_directory/</code>.</li>
<li>A path or url to a <em>pytorch index checkpoint file</em> (e.g, <code>./pt_model/</code>). In this case,
<code>encoder_from_pt</code> should be set to <code>True</code>.</li>
</ul>`,name:"encoder_pretrained_model_name_or_path"},{anchor:"transformers.TFVisionEncoderDecoderModel.from_encoder_decoder_pretrained.decoder_pretrained_model_name_or_path",description:`<strong>decoder_pretrained_model_name_or_path</strong> (<code>str</code>, <em>optional</em>, defaults to <em>None</em>) &#x2014;
Information necessary to initiate the decoder. Can be either:</p>
<ul>
<li>A string, the <em>model id</em> of a pretrained model hosted inside a model repo on huggingface.co.
Valid model ids can be located at the root-level, like <code>bert-base-uncased</code>, or namespaced under a
user or organization name, like <code>dbmdz/bert-base-german-cased</code>.</li>
<li>A path to a <em>directory</em> containing model weights saved using
<a href="/docs/transformers/pr_15839/en/main_classes/model#transformers.TFPreTrainedModel.save_pretrained">save_pretrained()</a>, e.g., <code>./my_model_directory/</code>.</li>
<li>A path or url to a <em>pytorch checkpoint file</em> (e.g, <code>./pt_model/</code>). In this case,
<code>decoder_from_pt</code> should be set to <code>True</code>.</li>
</ul>`,name:"decoder_pretrained_model_name_or_path"},{anchor:"transformers.TFVisionEncoderDecoderModel.from_encoder_decoder_pretrained.model_args",description:`<strong>model_args</strong> (remaining positional arguments, <em>optional</em>) &#x2014;
All remaning positional arguments will be passed to the underlying model&#x2019;s <code>__init__</code> method.`,name:"model_args"},{anchor:"transformers.TFVisionEncoderDecoderModel.from_encoder_decoder_pretrained.kwargs",description:`<strong>kwargs</strong> (remaining dictionary of keyword arguments, <em>optional</em>) &#x2014;
Can be used to update the configuration object (after it being loaded) and initiate the model (e.g.,
<code>output_attentions=True</code>).</p>
<ul>
<li>To update the encoder configuration, use the prefix <em>encoder_</em> for each configuration parameter.</li>
<li>To update the decoder configuration, use the prefix <em>decoder_</em> for each configuration parameter.</li>
<li>To update the parent model configuration, do not use a prefix for each configuration parameter.</li>
</ul>
<p>Behaves differently depending on whether a <code>config</code> is provided or automatically loaded.`,name:"kwargs"}]}}),no=new yo({props:{code:`from transformers import TFVisionEncoderDecoderModel

# initialize a vit-bert from a pretrained ViT and a pretrained BERT model. Note that the cross-attention layers will be randomly initialized
model = TFVisionEncoderDecoderModel.from_encoder_decoder_pretrained(
    "google/vit-base-patch16-224-in21k", "bert-base-uncased"
)
# saving model after fine-tuning
model.save_pretrained("./vit-bert")
# load fine-tuned model
model = TFVisionEncoderDecoderModel.from_pretrained("./vit-bert")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFVisionEncoderDecoderModel

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># initialize a vit-bert from a pretrained ViT and a pretrained BERT model. Note that the cross-attention layers will be randomly initialized</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = TFVisionEncoderDecoderModel.from_encoder_decoder_pretrained(
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;google/vit-base-patch16-224-in21k&quot;</span>, <span class="hljs-string">&quot;bert-base-uncased&quot;</span>
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># saving model after fine-tuning</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model.save_pretrained(<span class="hljs-string">&quot;./vit-bert&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># load fine-tuned model</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = TFVisionEncoderDecoderModel.from_pretrained(<span class="hljs-string">&quot;./vit-bert&quot;</span>)`}}),ro=new Yt({}),ao=new B({props:{name:"class transformers.FlaxVisionEncoderDecoderModel",anchor:"transformers.FlaxVisionEncoderDecoderModel",parameters:[{name:"config",val:": VisionEncoderDecoderConfig"},{name:"input_shape",val:": typing.Optional[typing.Tuple] = None"},{name:"seed",val:": int = 0"},{name:"dtype",val:": dtype = <class 'jax._src.numpy.lax_numpy.float32'>"},{name:"**kwargs",val:""}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/models/vision_encoder_decoder/modeling_flax_vision_encoder_decoder.py#L268",parametersDescription:[{anchor:"transformers.FlaxVisionEncoderDecoderModel.config",description:`<strong>config</strong> (<a href="/docs/transformers/pr_15839/en/model_doc/vision-encoder-decoder#transformers.VisionEncoderDecoderConfig">VisionEncoderDecoderConfig</a>) &#x2014; Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
configuration. Check out the <a href="/docs/transformers/pr_15839/en/main_classes/model#transformers.FlaxPreTrainedModel.from_pretrained">from_pretrained()</a> method to load the model weights.`,name:"config"},{anchor:"transformers.FlaxVisionEncoderDecoderModel.dtype",description:`<strong>dtype</strong> (<code>jax.numpy.dtype</code>, <em>optional</em>, defaults to <code>jax.numpy.float32</code>) &#x2014;
The data type of the computation. Can be one of <code>jax.numpy.float32</code>, <code>jax.numpy.float16</code> (on GPUs) and
<code>jax.numpy.bfloat16</code> (on TPUs).</p>
<p>This can be used to enable mixed-precision training or half-precision inference on GPUs or TPUs. If
specified all the computation will be performed with the given <code>dtype</code>.</p>
<p><strong>Note that this only specifies the dtype of the computation and does not influence the dtype of model
parameters.</strong></p>
<p>If you wish to change the dtype of the model parameters, see <a href="/docs/transformers/pr_15839/en/main_classes/model#transformers.FlaxPreTrainedModel.to_fp16">to_fp16()</a> and
<a href="/docs/transformers/pr_15839/en/main_classes/model#transformers.FlaxPreTrainedModel.to_bf16">to_bf16()</a>.`,name:"dtype"}]}}),fo=new B({props:{name:"__call__",anchor:"transformers.FlaxVisionEncoderDecoderModel.__call__",parameters:[{name:"pixel_values",val:": ndarray"},{name:"decoder_input_ids",val:": typing.Optional[jax._src.numpy.lax_numpy.ndarray] = None"},{name:"decoder_attention_mask",val:": typing.Optional[jax._src.numpy.lax_numpy.ndarray] = None"},{name:"decoder_position_ids",val:": typing.Optional[jax._src.numpy.lax_numpy.ndarray] = None"},{name:"output_attentions",val:": typing.Optional[bool] = None"},{name:"output_hidden_states",val:": typing.Optional[bool] = None"},{name:"return_dict",val:": typing.Optional[bool] = None"},{name:"train",val:": bool = False"},{name:"params",val:": dict = None"},{name:"dropout_rng",val:": PRNGKey = None"}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/models/vision_encoder_decoder/modeling_flax_vision_encoder_decoder.py#L583",parametersDescription:[{anchor:"transformers.FlaxVisionEncoderDecoderModel.__call__.pixel_values",description:`<strong>pixel_values</strong> (<code>jnp.ndarray</code> of shape <code>(batch_size, num_channels, height, width)</code>) &#x2014;
Pixel values. Pixel values can be obtained using the vision model&#x2019;s feature extractor. For example, using
<a href="/docs/transformers/pr_15839/en/model_doc/vit#transformers.ViTFeatureExtractor">ViTFeatureExtractor</a>. See <a href="/docs/transformers/pr_15839/en/model_doc/vit#transformers.ViTFeatureExtractor.__call__">ViTFeatureExtractor.<strong>call</strong>()</a> for details.`,name:"pixel_values"},{anchor:"transformers.FlaxVisionEncoderDecoderModel.__call__.decoder_input_ids",description:`<strong>decoder_input_ids</strong> (<code>jnp.ndarray</code> of shape <code>(batch_size, target_sequence_length)</code>, <em>optional</em>) &#x2014;
Indices of decoder input sequence tokens in the vocabulary.</p>
<p>Indices can be obtained using <a href="/docs/transformers/pr_15839/en/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a>. See <a href="/docs/transformers/pr_15839/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.encode">PreTrainedTokenizer.encode()</a> and
<a href="/docs/transformers/pr_15839/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details.</p>
<p><a href="../glossary#decoder-input-ids">What are decoder input IDs?</a>`,name:"decoder_input_ids"},{anchor:"transformers.FlaxVisionEncoderDecoderModel.__call__.decoder_attention_mask",description:`<strong>decoder_attention_mask</strong> (<code>jnp.ndarray</code> of shape <code>(batch_size, target_sequence_length)</code>, <em>optional</em>) &#x2014;
Default behavior: generate a tensor that ignores pad tokens in <code>decoder_input_ids</code>. Causal mask will also
be used by default.`,name:"decoder_attention_mask"},{anchor:"transformers.FlaxVisionEncoderDecoderModel.__call__.decoder_position_ids",description:`<strong>decoder_position_ids</strong> (<code>jnp.ndarray</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the
range <code>[0, config.decoder.max_position_embeddings - 1]</code>.`,name:"decoder_position_ids"},{anchor:"transformers.FlaxVisionEncoderDecoderModel.__call__.output_attentions",description:`<strong>output_attentions</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the attentions tensors of all attention layers. See <code>attentions</code> under returned
tensors for more detail.`,name:"output_attentions"},{anchor:"transformers.FlaxVisionEncoderDecoderModel.__call__.output_hidden_states",description:`<strong>output_hidden_states</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the hidden states of all layers. See <code>hidden_states</code> under returned tensors for
more detail.`,name:"output_hidden_states"},{anchor:"transformers.FlaxVisionEncoderDecoderModel.__call__.return_dict",description:`<strong>return_dict</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
If set to <code>True</code>, the model will return a <code>FlaxSeq2SeqLMOutput</code> instead of a plain tuple.`,name:"return_dict"}],returnDescription:`
<p>A <a
  href="/docs/transformers/pr_15839/en/main_classes/output#transformers.modeling_flax_outputs.FlaxSeq2SeqLMOutput"
>transformers.modeling_flax_outputs.FlaxSeq2SeqLMOutput</a> or a tuple of
<code>torch.FloatTensor</code> (if <code>return_dict=False</code> is passed or when <code>config.return_dict=False</code>) comprising various
elements depending on the configuration (<a
  href="/docs/transformers/pr_15839/en/model_doc/vision-encoder-decoder#transformers.VisionEncoderDecoderConfig"
>VisionEncoderDecoderConfig</a>) and inputs.</p>
<ul>
<li>
<p><strong>logits</strong> (<code>jnp.ndarray</code> of shape <code>(batch_size, sequence_length, config.vocab_size)</code>) \u2014 Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).</p>
</li>
<li>
<p><strong>past_key_values</strong> (<code>tuple(tuple(jnp.ndarray))</code>, <em>optional</em>, returned when <code>use_cache=True</code> is passed or when <code>config.use_cache=True</code>) \u2014 Tuple of <code>tuple(jnp.ndarray)</code> of length <code>config.n_layers</code>, with each tuple having 2 tensors of shape
<code>(batch_size, num_heads, sequence_length, embed_size_per_head)</code>) and 2 additional tensors of shape
<code>(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)</code>.</p>
<p>Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
blocks) that can be used (see <code>past_key_values</code> input) to speed up sequential decoding.</p>
</li>
<li>
<p><strong>decoder_hidden_states</strong> (<code>tuple(jnp.ndarray)</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) \u2014 Tuple of <code>jnp.ndarray</code> (one for the output of the embeddings + one for the output of each layer) of shape
<code>(batch_size, sequence_length, hidden_size)</code>.</p>
<p>Hidden-states of the decoder at the output of each layer plus the initial embedding outputs.</p>
</li>
<li>
<p><strong>decoder_attentions</strong> (<code>tuple(jnp.ndarray)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) \u2014 Tuple of <code>jnp.ndarray</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p>
<p>Attentions weights of the decoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.</p>
</li>
<li>
<p><strong>cross_attentions</strong> (<code>tuple(jnp.ndarray)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) \u2014 Tuple of <code>jnp.ndarray</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p>
<p>Attentions weights of the decoder\u2019s cross-attention layer, after the attention softmax, used to compute the
weighted average in the cross-attention heads.</p>
</li>
<li>
<p><strong>encoder_last_hidden_state</strong> (<code>jnp.ndarray</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>, <em>optional</em>) \u2014 Sequence of hidden-states at the output of the last layer of the encoder of the model.</p>
</li>
<li>
<p><strong>encoder_hidden_states</strong> (<code>tuple(jnp.ndarray)</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) \u2014 Tuple of <code>jnp.ndarray</code> (one for the output of the embeddings + one for the output of each layer) of shape
<code>(batch_size, sequence_length, hidden_size)</code>.</p>
<p>Hidden-states of the encoder at the output of each layer plus the initial embedding outputs.</p>
</li>
<li>
<p><strong>encoder_attentions</strong> (<code>tuple(jnp.ndarray)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) \u2014 Tuple of <code>jnp.ndarray</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p>
<p>Attentions weights of the encoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.</p>
</li>
</ul>
`,returnType:`
<p><a
  href="/docs/transformers/pr_15839/en/main_classes/output#transformers.modeling_flax_outputs.FlaxSeq2SeqLMOutput"
>transformers.modeling_flax_outputs.FlaxSeq2SeqLMOutput</a> or <code>tuple(torch.FloatTensor)</code></p>
`}}),ye=new Vd({props:{$$slots:{default:[lc]},$$scope:{ctx:Q}}}),uo=new yo({props:{code:`from transformers import FlaxVisionEncoderDecoderModel, ViTFeatureExtractor, GPT2Tokenizer
from PIL import Image
import requests

url = "http://images.cocodataset.org/val2017/000000039769.jpg"
image = Image.open(requests.get(url, stream=True).raw)

feature_extractor = ViTFeatureExtractor.from_pretrained("google/vit-base-patch16-224-in21k")

# load output tokenizer
tokenizer_output = GPT2Tokenizer.from_pretrained("gpt2")

# initialize a vit-gpt2 from pretrained ViT and GPT2 models. Note that the cross-attention layers will be randomly initialized
model = FlaxVisionEncoderDecoderModel.from_encoder_decoder_pretrained(
    "google/vit-base-patch16-224-in21k", "gpt2"
)

pixel_values = feature_extractor(images=image, return_tensors="np").pixel_values

# use GPT2's eos_token as the pad as well as eos token
model.config.eos_token_id = model.config.decoder.eos_token_id
model.config.pad_token_id = model.config.eos_token_id

# generation
sequences = model.generate(pixel_values, num_beams=4, max_length=12).sequences

captions = tokenizer_output.batch_decode(sequences, skip_special_tokens=True)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> FlaxVisionEncoderDecoderModel, ViTFeatureExtractor, GPT2Tokenizer
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> requests

<span class="hljs-meta">&gt;&gt;&gt; </span>url = <span class="hljs-string">&quot;http://images.cocodataset.org/val2017/000000039769.jpg&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>image = Image.<span class="hljs-built_in">open</span>(requests.get(url, stream=<span class="hljs-literal">True</span>).raw)

<span class="hljs-meta">&gt;&gt;&gt; </span>feature_extractor = ViTFeatureExtractor.from_pretrained(<span class="hljs-string">&quot;google/vit-base-patch16-224-in21k&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># load output tokenizer</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer_output = GPT2Tokenizer.from_pretrained(<span class="hljs-string">&quot;gpt2&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># initialize a vit-gpt2 from pretrained ViT and GPT2 models. Note that the cross-attention layers will be randomly initialized</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = FlaxVisionEncoderDecoderModel.from_encoder_decoder_pretrained(
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;google/vit-base-patch16-224-in21k&quot;</span>, <span class="hljs-string">&quot;gpt2&quot;</span>
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>pixel_values = feature_extractor(images=image, return_tensors=<span class="hljs-string">&quot;np&quot;</span>).pixel_values

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># use GPT2&#x27;s eos_token as the pad as well as eos token</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model.config.eos_token_id = model.config.decoder.eos_token_id
<span class="hljs-meta">&gt;&gt;&gt; </span>model.config.pad_token_id = model.config.eos_token_id

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># generation</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>sequences = model.generate(pixel_values, num_beams=<span class="hljs-number">4</span>, max_length=<span class="hljs-number">12</span>).sequences

<span class="hljs-meta">&gt;&gt;&gt; </span>captions = tokenizer_output.batch_decode(sequences, skip_special_tokens=<span class="hljs-literal">True</span>)`}}),go=new B({props:{name:"from_encoder_decoder_pretrained",anchor:"transformers.FlaxVisionEncoderDecoderModel.from_encoder_decoder_pretrained",parameters:[{name:"encoder_pretrained_model_name_or_path",val:": typing.Union[str, os.PathLike, NoneType] = None"},{name:"decoder_pretrained_model_name_or_path",val:": typing.Union[str, os.PathLike, NoneType] = None"},{name:"*model_args",val:""},{name:"**kwargs",val:""}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/models/vision_encoder_decoder/modeling_flax_vision_encoder_decoder.py#L708",parametersDescription:[{anchor:"transformers.FlaxVisionEncoderDecoderModel.from_encoder_decoder_pretrained.encoder_pretrained_model_name_or_path",description:`<strong>encoder_pretrained_model_name_or_path</strong> (<code>Union[str, os.PathLike]</code>, <em>optional</em>) &#x2014;
Information necessary to initiate the encoder. Can be either:</p>
<ul>
<li>A string, the <em>model id</em> of a pretrained model hosted inside a model repo on huggingface.co. An
example is <code>google/vit-base-patch16-224-in21k</code>.</li>
<li>A path to a <em>directory</em> containing model weights saved using
<a href="/docs/transformers/pr_15839/en/main_classes/model#transformers.FlaxPreTrainedModel.save_pretrained">save_pretrained()</a>, e.g., <code>./my_model_directory/</code>.</li>
</ul>`,name:"encoder_pretrained_model_name_or_path"},{anchor:"transformers.FlaxVisionEncoderDecoderModel.from_encoder_decoder_pretrained.decoder_pretrained_model_name_or_path",description:`<strong>decoder_pretrained_model_name_or_path</strong> (<code>Union[str, os.PathLike]</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Information necessary to initiate the decoder. Can be either:</p>
<ul>
<li>A string, the <em>model id</em> of a pretrained model hosted inside a model repo on huggingface.co.
Valid model ids can be located at the root-level, like <code>bert-base-uncased</code>, or namespaced under a
user or organization name, like <code>dbmdz/bert-base-german-cased</code>.</li>
<li>A path to a <em>directory</em> containing model weights saved using
<a href="/docs/transformers/pr_15839/en/main_classes/model#transformers.FlaxPreTrainedModel.save_pretrained">save_pretrained()</a>, e.g., <code>./my_model_directory/</code>.</li>
</ul>`,name:"decoder_pretrained_model_name_or_path"},{anchor:"transformers.FlaxVisionEncoderDecoderModel.from_encoder_decoder_pretrained.model_args",description:`<strong>model_args</strong> (remaining positional arguments, <em>optional</em>) &#x2014;
All remaning positional arguments will be passed to the underlying model&#x2019;s <code>__init__</code> method.`,name:"model_args"},{anchor:"transformers.FlaxVisionEncoderDecoderModel.from_encoder_decoder_pretrained.kwargs",description:`<strong>kwargs</strong> (remaining dictionary of keyword arguments, <em>optional</em>) &#x2014;
Can be used to update the configuration object (after it being loaded) and initiate the model (e.g.,
<code>output_attentions=True</code>).</p>
<ul>
<li>To update the encoder configuration, use the prefix <em>encoder_</em> for each configuration parameter.</li>
<li>To update the decoder configuration, use the prefix <em>decoder_</em> for each configuration parameter.</li>
<li>To update the parent model configuration, do not use a prefix for each configuration parameter.</li>
</ul>
<p>Behaves differently depending on whether a <code>config</code> is provided or automatically loaded.`,name:"kwargs"}]}}),_o=new yo({props:{code:`from transformers import FlaxVisionEncoderDecoderModel

# initialize a vit-gpt2 from a pretrained ViT and a pretrained GPT2 model. Note that the cross-attention layers will be randomly initialized
model = FlaxVisionEncoderDecoderModel.from_encoder_decoder_pretrained(
    "google/vit-base-patch16-224-in21k", "gpt2"
)
# saving model after fine-tuning
model.save_pretrained("./vit-gpt2")
# load fine-tuned model
model = FlaxVisionEncoderDecoderModel.from_pretrained("./vit-gpt2")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> FlaxVisionEncoderDecoderModel

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># initialize a vit-gpt2 from a pretrained ViT and a pretrained GPT2 model. Note that the cross-attention layers will be randomly initialized</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = FlaxVisionEncoderDecoderModel.from_encoder_decoder_pretrained(
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;google/vit-base-patch16-224-in21k&quot;</span>, <span class="hljs-string">&quot;gpt2&quot;</span>
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># saving model after fine-tuning</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model.save_pretrained(<span class="hljs-string">&quot;./vit-gpt2&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># load fine-tuned model</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = FlaxVisionEncoderDecoderModel.from_pretrained(<span class="hljs-string">&quot;./vit-gpt2&quot;</span>)`}}),{c(){p=r("meta"),P=c(),m=r("h1"),M=r("a"),F=r("span"),u(y.$$.fragment),j=c(),C=r("span"),Vn=t("Vision Encoder Decoder Models"),Jt=c(),f=r("p"),$n=t("The "),Eo=r("a"),qn=t("VisionEncoderDecoderModel"),zn=t(` can be used to initialize an image-to-text-sequence model with any
pretrained Transformer-based vision autoencoding model as the encoder (`),rt=r("em"),Pn=t("e.g."),Fn=c(),ko=r("a"),Cn=t("ViT"),An=t(", "),xo=r("a"),Sn=t("BEiT"),In=t(", "),jo=r("a"),Ln=t("DeiT"),Nn=t(", "),Do=r("a"),On=t("Swin"),Rn=t(`)
and any pretrained language model as the decoder (`),at=r("em"),Bn=t("e.g."),Gn=c(),Mo=r("a"),Wn=t("RoBERTa"),Un=t(", "),Vo=r("a"),Hn=t("GPT2"),Zn=t(", "),$o=r("a"),Yn=t("BERT"),Jn=t(", "),qo=r("a"),Kn=t("DistilBERT"),Qn=t(")."),Kt=c(),he=r("p"),Xn=t(`The effectiveness of initializing image-to-text-sequence models with pretrained checkpoints has been shown in (for
example) `),xe=r("a"),er=t("TrOCR: Transformer-based Optical Character Recognition with Pre-trained Models"),or=t(` by Minghao Li, Tengchao Lv, Lei Cui, Yijuan Lu, Dinei Florencio, Cha Zhang,
Zhoujun Li, Furu Wei.`),Qt=c(),G=r("p"),tr=t("An example of how to use a "),zo=r("a"),nr=t("VisionEncoderDecoderModel"),rr=t(" for inference can be seen in "),Po=r("a"),ar=t("TrOCR"),sr=t("."),Xt=c(),X=r("h2"),me=r("a"),st=r("span"),u(je.$$.fragment),dr=c(),dt=r("span"),ir=t("VisionEncoderDecoderConfig"),en=c(),z=r("div"),u(De.$$.fragment),cr=c(),fe=r("p"),Fo=r("a"),lr=t("VisionEncoderDecoderConfig"),pr=t(` is the configuration class to store the configuration of a
`),Co=r("a"),hr=t("VisionEncoderDecoderModel"),mr=t(`. It is used to instantiate a Vision-Encoder-Text-Decoder model according to the
specified arguments, defining the encoder and decoder configs.`),fr=c(),ee=r("p"),ur=t("Configuration objects inherit from "),Ao=r("a"),gr=t("PretrainedConfig"),_r=t(` and can be used to control the model outputs. Read the
documentation from `),So=r("a"),vr=t("PretrainedConfig"),br=t(" for more information."),Tr=c(),it=r("p"),wr=t("Examples:"),yr=c(),u(Me.$$.fragment),Er=c(),ue=r("div"),u(Ve.$$.fragment),kr=c(),$e=r("p"),xr=t("Instantiate a "),Io=r("a"),jr=t("VisionEncoderDecoderConfig"),Dr=t(` (or a derived class) from a pre-trained encoder model
configuration and decoder model configuration.`),Mr=c(),ge=r("div"),u(qe.$$.fragment),Vr=c(),oe=r("p"),$r=t("Serializes this instance to a Python dictionary. Override the default "),ct=r("em"),qr=t("to_dict()"),zr=t(" from "),lt=r("em"),Pr=t("PretrainedConfig"),Fr=t("."),on=c(),te=r("h2"),_e=r("a"),pt=r("span"),u(ze.$$.fragment),Cr=c(),ht=r("span"),Ar=t("VisionEncoderDecoderModel"),tn=c(),E=r("div"),u(Pe.$$.fragment),Sr=c(),ne=r("p"),Ir=t(`This class can be used to initialize an image-to-text-sequence model with any pretrained vision autoencoding model
as the encoder and any pretrained text autoregressive model as the decoder. The encoder is loaded via
`),mt=r("code"),Lr=t("from_pretrained()"),Nr=t("function and the decoder is loaded via "),ft=r("code"),Or=t("from_pretrained()"),Rr=t(`function. Cross-attention layers are automatically added to the decoder and should be fine-tuned on a downstream
generative task, like image captioning.`),Br=c(),Fe=r("p"),Gr=t(`The effectiveness of initializing sequence-to-sequence models with pretrained checkpoints for sequence generation
tasks was shown in `),Ce=r("a"),Wr=t(`Leveraging Pre-trained Checkpoints for Sequence Generation
Tasks`),Ur=t(` by Sascha Rothe, Shashi Narayan, Aliaksei Severyn. Michael Matena, Yanqi
Zhou, Wei Li, Peter J. Liu.`),Hr=c(),Ae=r("p"),Zr=t("Additionally, in "),Se=r("a"),Yr=t(`TrOCR: Transformer-based Optical Character Recognition with Pre-trained
Models`),Jr=t(` it is shown how leveraging large pretrained vision models for optical
character recognition (OCR) yields a significant performance improvement.`),Kr=c(),ut=r("p"),Qr=t(`After such a Vision-Encoder-Text-Decoder model has been trained/fine-tuned, it can be saved/loaded just like any
other models (see the examples for more information).`),Xr=c(),Ie=r("p"),ea=t("This model inherits from "),Lo=r("a"),oa=t("PreTrainedModel"),ta=t(`. Check the superclass documentation for the generic methods the
library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
etc.)`),na=c(),Le=r("p"),ra=t("This model is also a PyTorch "),Ne=r("a"),aa=t("torch.nn.Module"),sa=t(` subclass.
Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
and behavior.`),da=c(),W=r("p"),No=r("a"),ia=t("VisionEncoderDecoderModel"),ca=t(` is a generic model class that will be instantiated as a transformer architecture with
one of the base vision model classes of the library as encoder and another one as decoder when created with the
:meth`),gt=r("em"),la=t("~transformers.AutoModel.from_pretrained"),pa=t(` class method for the encoder and
:meth`),_t=r("em"),ha=t("~transformers.AutoModelForCausalLM.from_pretrained"),ma=t(" class method for the decoder."),fa=c(),A=r("div"),u(Oe.$$.fragment),ua=c(),re=r("p"),ga=t("The "),Oo=r("a"),_a=t("VisionEncoderDecoderModel"),va=t(" forward method, overrides the "),vt=r("code"),ba=t("__call__"),Ta=t(" special method."),wa=c(),u(ve.$$.fragment),ya=c(),bt=r("p"),Ea=t("Examples:"),ka=c(),u(Re.$$.fragment),xa=c(),S=r("div"),u(Be.$$.fragment),ja=c(),Tt=r("p"),Da=t(`Instantiate an encoder and a decoder from one or two base classes of the library from pretrained model
checkpoints.`),Ma=c(),ae=r("p"),Va=t("The model is set in evaluation mode by default using "),wt=r("code"),$a=t("model.eval()"),qa=t(` (Dropout modules are deactivated). To train
the model, you need to first set it back in training mode with `),yt=r("code"),za=t("model.train()"),Pa=t("."),Fa=c(),Et=r("p"),Ca=t("Example:"),Aa=c(),u(Ge.$$.fragment),nn=c(),se=r("h2"),be=r("a"),kt=r("span"),u(We.$$.fragment),Sa=c(),xt=r("span"),Ia=t("TFVisionEncoderDecoderModel"),rn=c(),k=r("div"),u(Ue.$$.fragment),La=c(),de=r("p"),Na=t(`This class can be used to initialize an image-to-text-sequence model with any pretrained vision autoencoding model
as the encoder and any pretrained text autoregressive model as the decoder. The encoder is loaded via
`),jt=r("code"),Oa=t("from_pretrained()"),Ra=t("function and the decoder is loaded via "),Dt=r("code"),Ba=t("from_pretrained()"),Ga=t(`function. Cross-attention layers are automatically added to the decoder and should be fine-tuned on a downstream
generative task, like image captioning.`),Wa=c(),He=r("p"),Ua=t(`The effectiveness of initializing sequence-to-sequence models with pretrained checkpoints for sequence generation
tasks was shown in `),Ze=r("a"),Ha=t(`Leveraging Pre-trained Checkpoints for Sequence Generation
Tasks`),Za=t(` by Sascha Rothe, Shashi Narayan, Aliaksei Severyn. Michael Matena, Yanqi
Zhou, Wei Li, Peter J. Liu.`),Ya=c(),Ye=r("p"),Ja=t("Additionally, in "),Je=r("a"),Ka=t(`TrOCR: Transformer-based Optical Character Recognition with Pre-trained
Models`),Qa=t(` it is shown how leveraging large pretrained vision models for optical
character recognition (OCR) yields a significant performance improvement.`),Xa=c(),Mt=r("p"),es=t(`After such a Vision-Encoder-Text-Decoder model has been trained/fine-tuned, it can be saved/loaded just like any
other models (see the examples for more information).`),os=c(),Ke=r("p"),ts=t("This model inherits from "),Ro=r("a"),ns=t("TFPreTrainedModel"),rs=t(`. Check the superclass documentation for the generic methods the
library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
etc.)`),as=c(),Qe=r("p"),ss=t("This model is also a "),Xe=r("a"),ds=t("tf.keras.Model"),is=t(` subclass. Use it
as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
behavior.`),cs=c(),U=r("p"),Bo=r("a"),ls=t("TFVisionEncoderDecoderModel"),ps=t(` is a generic model class that will be instantiated as a transformer architecture
with one of the base vision model classes of the library as encoder and another one of the base model classes as
decoder when created with the `),Vt=r("code"),hs=t("from_pretrained()"),ms=t(`class method for the encoder and
`),$t=r("code"),fs=t("from_pretrained()"),us=t("class method for the decoder."),gs=c(),I=r("div"),u(eo.$$.fragment),_s=c(),ie=r("p"),vs=t("The "),Go=r("a"),bs=t("TFVisionEncoderDecoderModel"),Ts=t(" forward method, overrides the "),qt=r("code"),ws=t("__call__"),ys=t(" special method."),Es=c(),u(Te.$$.fragment),ks=c(),zt=r("p"),xs=t("Examples:"),js=c(),u(oo.$$.fragment),Ds=c(),O=r("div"),u(to.$$.fragment),Ms=c(),Pt=r("p"),Vs=t(`Instantiate an encoder and a decoder from one or two base classes of the library from pretrained model
checkpoints.`),$s=c(),Ft=r("p"),qs=t("Example:"),zs=c(),u(no.$$.fragment),an=c(),ce=r("h2"),we=r("a"),Ct=r("span"),u(ro.$$.fragment),Ps=c(),At=r("span"),Fs=t("FlaxVisionEncoderDecoderModel"),sn=c(),x=r("div"),u(ao.$$.fragment),Cs=c(),le=r("p"),As=t(`This class can be used to initialize an image-to-text-sequence model with any pretrained vision autoencoding model
as the encoder and any pretrained text autoregressive model as the decoder. The encoder is loaded via
`),St=r("code"),Ss=t("from_pretrained()"),Is=t("function and the decoder is loaded via "),It=r("code"),Ls=t("from_pretrained()"),Ns=t(`function. Cross-attention layers are automatically added to the decoder and should be fine-tuned on a downstream
generative task, like image captioning.`),Os=c(),so=r("p"),Rs=t(`The effectiveness of initializing sequence-to-sequence models with pretrained checkpoints for sequence generation
tasks was shown in `),io=r("a"),Bs=t(`Leveraging Pre-trained Checkpoints for Sequence Generation
Tasks`),Gs=t(` by Sascha Rothe, Shashi Narayan, Aliaksei Severyn. Michael Matena, Yanqi
Zhou, Wei Li, Peter J. Liu.`),Ws=c(),co=r("p"),Us=t("Additionally, in "),lo=r("a"),Hs=t(`TrOCR: Transformer-based Optical Character Recognition with Pre-trained
Models`),Zs=t(` it is shown how leveraging large pretrained vision models for optical
character recognition (OCR) yields a significant performance improvement.`),Ys=c(),Lt=r("p"),Js=t(`After such a Vision-Encoder-Text-Decoder model has been trained/fine-tuned, it can be saved/loaded just like any
other models (see the examples for more information).`),Ks=c(),po=r("p"),Qs=t("This model inherits from "),Wo=r("a"),Xs=t("FlaxPreTrainedModel"),ed=t(`. Check the superclass documentation for the generic methods the
library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
etc.)`),od=c(),ho=r("p"),td=t(`This model is also a Flax Linen
`),mo=r("a"),nd=t("flax.nn.Module"),rd=t(` subclass. Use it as a
regular Flax Module and refer to the Flax documentation for all matter related to general usage and behavior.`),ad=c(),H=r("p"),Uo=r("a"),sd=t("FlaxVisionEncoderDecoderModel"),dd=t(` is a generic model class that will be instantiated as a transformer architecture
with the module (flax.nn.Module) of one of the base vision model classes of the library as encoder module and
another one as decoder module when created with the :meth`),Nt=r("em"),id=t("~transformers.FlaxAutoModel.from_pretrained"),cd=t(` class method
for the encoder and :meth`),Ot=r("em"),ld=t("~transformers.FlaxAutoModelForCausalLM.from_pretrained"),pd=t(" class method for the decoder."),hd=c(),L=r("div"),u(fo.$$.fragment),md=c(),pe=r("p"),fd=t("The "),Ho=r("a"),ud=t("FlaxVisionEncoderDecoderModel"),gd=t(" forward method, overrides the "),Rt=r("code"),_d=t("__call__"),vd=t(" special method."),bd=c(),u(ye.$$.fragment),Td=c(),Bt=r("p"),wd=t("Examples:"),yd=c(),u(uo.$$.fragment),Ed=c(),R=r("div"),u(go.$$.fragment),kd=c(),Gt=r("p"),xd=t(`Instantiate an encoder and a decoder from one or two base classes of the library from pretrained model
checkpoints.`),jd=c(),Wt=r("p"),Dd=t("Example:"),Md=c(),u(_o.$$.fragment),this.h()},l(d){const h=dc('[data-svelte="svelte-1phssyn"]',document.head);p=a(h,"META",{name:!0,content:!0}),h.forEach(o),P=l(d),m=a(d,"H1",{class:!0});var vo=s(m);M=a(vo,"A",{id:!0,class:!0,href:!0});var Ut=s(M);F=a(Ut,"SPAN",{});var Ht=s(F);g(y.$$.fragment,Ht),Ht.forEach(o),Ut.forEach(o),j=l(vo),C=a(vo,"SPAN",{});var $d=s(C);Vn=n($d,"Vision Encoder Decoder Models"),$d.forEach(o),vo.forEach(o),Jt=l(d),f=a(d,"P",{});var D=s(f);$n=n(D,"The "),Eo=a(D,"A",{href:!0});var qd=s(Eo);qn=n(qd,"VisionEncoderDecoderModel"),qd.forEach(o),zn=n(D,` can be used to initialize an image-to-text-sequence model with any
pretrained Transformer-based vision autoencoding model as the encoder (`),rt=a(D,"EM",{});var zd=s(rt);Pn=n(zd,"e.g."),zd.forEach(o),Fn=l(D),ko=a(D,"A",{href:!0});var Pd=s(ko);Cn=n(Pd,"ViT"),Pd.forEach(o),An=n(D,", "),xo=a(D,"A",{href:!0});var Fd=s(xo);Sn=n(Fd,"BEiT"),Fd.forEach(o),In=n(D,", "),jo=a(D,"A",{href:!0});var Cd=s(jo);Ln=n(Cd,"DeiT"),Cd.forEach(o),Nn=n(D,", "),Do=a(D,"A",{href:!0});var Ad=s(Do);On=n(Ad,"Swin"),Ad.forEach(o),Rn=n(D,`)
and any pretrained language model as the decoder (`),at=a(D,"EM",{});var Sd=s(at);Bn=n(Sd,"e.g."),Sd.forEach(o),Gn=l(D),Mo=a(D,"A",{href:!0});var Id=s(Mo);Wn=n(Id,"RoBERTa"),Id.forEach(o),Un=n(D,", "),Vo=a(D,"A",{href:!0});var Ld=s(Vo);Hn=n(Ld,"GPT2"),Ld.forEach(o),Zn=n(D,", "),$o=a(D,"A",{href:!0});var Nd=s($o);Yn=n(Nd,"BERT"),Nd.forEach(o),Jn=n(D,", "),qo=a(D,"A",{href:!0});var Od=s(qo);Kn=n(Od,"DistilBERT"),Od.forEach(o),Qn=n(D,")."),D.forEach(o),Kt=l(d),he=a(d,"P",{});var cn=s(he);Xn=n(cn,`The effectiveness of initializing image-to-text-sequence models with pretrained checkpoints has been shown in (for
example) `),xe=a(cn,"A",{href:!0,rel:!0});var Rd=s(xe);er=n(Rd,"TrOCR: Transformer-based Optical Character Recognition with Pre-trained Models"),Rd.forEach(o),or=n(cn,` by Minghao Li, Tengchao Lv, Lei Cui, Yijuan Lu, Dinei Florencio, Cha Zhang,
Zhoujun Li, Furu Wei.`),cn.forEach(o),Qt=l(d),G=a(d,"P",{});var Zo=s(G);tr=n(Zo,"An example of how to use a "),zo=a(Zo,"A",{href:!0});var Bd=s(zo);nr=n(Bd,"VisionEncoderDecoderModel"),Bd.forEach(o),rr=n(Zo," for inference can be seen in "),Po=a(Zo,"A",{href:!0});var Gd=s(Po);ar=n(Gd,"TrOCR"),Gd.forEach(o),sr=n(Zo,"."),Zo.forEach(o),Xt=l(d),X=a(d,"H2",{class:!0});var ln=s(X);me=a(ln,"A",{id:!0,class:!0,href:!0});var Wd=s(me);st=a(Wd,"SPAN",{});var Ud=s(st);g(je.$$.fragment,Ud),Ud.forEach(o),Wd.forEach(o),dr=l(ln),dt=a(ln,"SPAN",{});var Hd=s(dt);ir=n(Hd,"VisionEncoderDecoderConfig"),Hd.forEach(o),ln.forEach(o),en=l(d),z=a(d,"DIV",{class:!0});var N=s(z);g(De.$$.fragment,N),cr=l(N),fe=a(N,"P",{});var Zt=s(fe);Fo=a(Zt,"A",{href:!0});var Zd=s(Fo);lr=n(Zd,"VisionEncoderDecoderConfig"),Zd.forEach(o),pr=n(Zt,` is the configuration class to store the configuration of a
`),Co=a(Zt,"A",{href:!0});var Yd=s(Co);hr=n(Yd,"VisionEncoderDecoderModel"),Yd.forEach(o),mr=n(Zt,`. It is used to instantiate a Vision-Encoder-Text-Decoder model according to the
specified arguments, defining the encoder and decoder configs.`),Zt.forEach(o),fr=l(N),ee=a(N,"P",{});var Yo=s(ee);ur=n(Yo,"Configuration objects inherit from "),Ao=a(Yo,"A",{href:!0});var Jd=s(Ao);gr=n(Jd,"PretrainedConfig"),Jd.forEach(o),_r=n(Yo,` and can be used to control the model outputs. Read the
documentation from `),So=a(Yo,"A",{href:!0});var Kd=s(So);vr=n(Kd,"PretrainedConfig"),Kd.forEach(o),br=n(Yo," for more information."),Yo.forEach(o),Tr=l(N),it=a(N,"P",{});var Qd=s(it);wr=n(Qd,"Examples:"),Qd.forEach(o),yr=l(N),g(Me.$$.fragment,N),Er=l(N),ue=a(N,"DIV",{class:!0});var pn=s(ue);g(Ve.$$.fragment,pn),kr=l(pn),$e=a(pn,"P",{});var hn=s($e);xr=n(hn,"Instantiate a "),Io=a(hn,"A",{href:!0});var Xd=s(Io);jr=n(Xd,"VisionEncoderDecoderConfig"),Xd.forEach(o),Dr=n(hn,` (or a derived class) from a pre-trained encoder model
configuration and decoder model configuration.`),hn.forEach(o),pn.forEach(o),Mr=l(N),ge=a(N,"DIV",{class:!0});var mn=s(ge);g(qe.$$.fragment,mn),Vr=l(mn),oe=a(mn,"P",{});var Jo=s(oe);$r=n(Jo,"Serializes this instance to a Python dictionary. Override the default "),ct=a(Jo,"EM",{});var ei=s(ct);qr=n(ei,"to_dict()"),ei.forEach(o),zr=n(Jo," from "),lt=a(Jo,"EM",{});var oi=s(lt);Pr=n(oi,"PretrainedConfig"),oi.forEach(o),Fr=n(Jo,"."),Jo.forEach(o),mn.forEach(o),N.forEach(o),on=l(d),te=a(d,"H2",{class:!0});var fn=s(te);_e=a(fn,"A",{id:!0,class:!0,href:!0});var ti=s(_e);pt=a(ti,"SPAN",{});var ni=s(pt);g(ze.$$.fragment,ni),ni.forEach(o),ti.forEach(o),Cr=l(fn),ht=a(fn,"SPAN",{});var ri=s(ht);Ar=n(ri,"VisionEncoderDecoderModel"),ri.forEach(o),fn.forEach(o),tn=l(d),E=a(d,"DIV",{class:!0});var V=s(E);g(Pe.$$.fragment,V),Sr=l(V),ne=a(V,"P",{});var Ko=s(ne);Ir=n(Ko,`This class can be used to initialize an image-to-text-sequence model with any pretrained vision autoencoding model
as the encoder and any pretrained text autoregressive model as the decoder. The encoder is loaded via
`),mt=a(Ko,"CODE",{});var ai=s(mt);Lr=n(ai,"from_pretrained()"),ai.forEach(o),Nr=n(Ko,"function and the decoder is loaded via "),ft=a(Ko,"CODE",{});var si=s(ft);Or=n(si,"from_pretrained()"),si.forEach(o),Rr=n(Ko,`function. Cross-attention layers are automatically added to the decoder and should be fine-tuned on a downstream
generative task, like image captioning.`),Ko.forEach(o),Br=l(V),Fe=a(V,"P",{});var un=s(Fe);Gr=n(un,`The effectiveness of initializing sequence-to-sequence models with pretrained checkpoints for sequence generation
tasks was shown in `),Ce=a(un,"A",{href:!0,rel:!0});var di=s(Ce);Wr=n(di,`Leveraging Pre-trained Checkpoints for Sequence Generation
Tasks`),di.forEach(o),Ur=n(un,` by Sascha Rothe, Shashi Narayan, Aliaksei Severyn. Michael Matena, Yanqi
Zhou, Wei Li, Peter J. Liu.`),un.forEach(o),Hr=l(V),Ae=a(V,"P",{});var gn=s(Ae);Zr=n(gn,"Additionally, in "),Se=a(gn,"A",{href:!0,rel:!0});var ii=s(Se);Yr=n(ii,`TrOCR: Transformer-based Optical Character Recognition with Pre-trained
Models`),ii.forEach(o),Jr=n(gn,` it is shown how leveraging large pretrained vision models for optical
character recognition (OCR) yields a significant performance improvement.`),gn.forEach(o),Kr=l(V),ut=a(V,"P",{});var ci=s(ut);Qr=n(ci,`After such a Vision-Encoder-Text-Decoder model has been trained/fine-tuned, it can be saved/loaded just like any
other models (see the examples for more information).`),ci.forEach(o),Xr=l(V),Ie=a(V,"P",{});var _n=s(Ie);ea=n(_n,"This model inherits from "),Lo=a(_n,"A",{href:!0});var li=s(Lo);oa=n(li,"PreTrainedModel"),li.forEach(o),ta=n(_n,`. Check the superclass documentation for the generic methods the
library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
etc.)`),_n.forEach(o),na=l(V),Le=a(V,"P",{});var vn=s(Le);ra=n(vn,"This model is also a PyTorch "),Ne=a(vn,"A",{href:!0,rel:!0});var pi=s(Ne);aa=n(pi,"torch.nn.Module"),pi.forEach(o),sa=n(vn,` subclass.
Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
and behavior.`),vn.forEach(o),da=l(V),W=a(V,"P",{});var bo=s(W);No=a(bo,"A",{href:!0});var hi=s(No);ia=n(hi,"VisionEncoderDecoderModel"),hi.forEach(o),ca=n(bo,` is a generic model class that will be instantiated as a transformer architecture with
one of the base vision model classes of the library as encoder and another one as decoder when created with the
:meth`),gt=a(bo,"EM",{});var mi=s(gt);la=n(mi,"~transformers.AutoModel.from_pretrained"),mi.forEach(o),pa=n(bo,` class method for the encoder and
:meth`),_t=a(bo,"EM",{});var fi=s(_t);ha=n(fi,"~transformers.AutoModelForCausalLM.from_pretrained"),fi.forEach(o),ma=n(bo," class method for the decoder."),bo.forEach(o),fa=l(V),A=a(V,"DIV",{class:!0});var Z=s(A);g(Oe.$$.fragment,Z),ua=l(Z),re=a(Z,"P",{});var Qo=s(re);ga=n(Qo,"The "),Oo=a(Qo,"A",{href:!0});var ui=s(Oo);_a=n(ui,"VisionEncoderDecoderModel"),ui.forEach(o),va=n(Qo," forward method, overrides the "),vt=a(Qo,"CODE",{});var gi=s(vt);ba=n(gi,"__call__"),gi.forEach(o),Ta=n(Qo," special method."),Qo.forEach(o),wa=l(Z),g(ve.$$.fragment,Z),ya=l(Z),bt=a(Z,"P",{});var _i=s(bt);Ea=n(_i,"Examples:"),_i.forEach(o),ka=l(Z),g(Re.$$.fragment,Z),Z.forEach(o),xa=l(V),S=a(V,"DIV",{class:!0});var Y=s(S);g(Be.$$.fragment,Y),ja=l(Y),Tt=a(Y,"P",{});var vi=s(Tt);Da=n(vi,`Instantiate an encoder and a decoder from one or two base classes of the library from pretrained model
checkpoints.`),vi.forEach(o),Ma=l(Y),ae=a(Y,"P",{});var Xo=s(ae);Va=n(Xo,"The model is set in evaluation mode by default using "),wt=a(Xo,"CODE",{});var bi=s(wt);$a=n(bi,"model.eval()"),bi.forEach(o),qa=n(Xo,` (Dropout modules are deactivated). To train
the model, you need to first set it back in training mode with `),yt=a(Xo,"CODE",{});var Ti=s(yt);za=n(Ti,"model.train()"),Ti.forEach(o),Pa=n(Xo,"."),Xo.forEach(o),Fa=l(Y),Et=a(Y,"P",{});var wi=s(Et);Ca=n(wi,"Example:"),wi.forEach(o),Aa=l(Y),g(Ge.$$.fragment,Y),Y.forEach(o),V.forEach(o),nn=l(d),se=a(d,"H2",{class:!0});var bn=s(se);be=a(bn,"A",{id:!0,class:!0,href:!0});var yi=s(be);kt=a(yi,"SPAN",{});var Ei=s(kt);g(We.$$.fragment,Ei),Ei.forEach(o),yi.forEach(o),Sa=l(bn),xt=a(bn,"SPAN",{});var ki=s(xt);Ia=n(ki,"TFVisionEncoderDecoderModel"),ki.forEach(o),bn.forEach(o),rn=l(d),k=a(d,"DIV",{class:!0});var $=s(k);g(Ue.$$.fragment,$),La=l($),de=a($,"P",{});var et=s(de);Na=n(et,`This class can be used to initialize an image-to-text-sequence model with any pretrained vision autoencoding model
as the encoder and any pretrained text autoregressive model as the decoder. The encoder is loaded via
`),jt=a(et,"CODE",{});var xi=s(jt);Oa=n(xi,"from_pretrained()"),xi.forEach(o),Ra=n(et,"function and the decoder is loaded via "),Dt=a(et,"CODE",{});var ji=s(Dt);Ba=n(ji,"from_pretrained()"),ji.forEach(o),Ga=n(et,`function. Cross-attention layers are automatically added to the decoder and should be fine-tuned on a downstream
generative task, like image captioning.`),et.forEach(o),Wa=l($),He=a($,"P",{});var Tn=s(He);Ua=n(Tn,`The effectiveness of initializing sequence-to-sequence models with pretrained checkpoints for sequence generation
tasks was shown in `),Ze=a(Tn,"A",{href:!0,rel:!0});var Di=s(Ze);Ha=n(Di,`Leveraging Pre-trained Checkpoints for Sequence Generation
Tasks`),Di.forEach(o),Za=n(Tn,` by Sascha Rothe, Shashi Narayan, Aliaksei Severyn. Michael Matena, Yanqi
Zhou, Wei Li, Peter J. Liu.`),Tn.forEach(o),Ya=l($),Ye=a($,"P",{});var wn=s(Ye);Ja=n(wn,"Additionally, in "),Je=a(wn,"A",{href:!0,rel:!0});var Mi=s(Je);Ka=n(Mi,`TrOCR: Transformer-based Optical Character Recognition with Pre-trained
Models`),Mi.forEach(o),Qa=n(wn,` it is shown how leveraging large pretrained vision models for optical
character recognition (OCR) yields a significant performance improvement.`),wn.forEach(o),Xa=l($),Mt=a($,"P",{});var Vi=s(Mt);es=n(Vi,`After such a Vision-Encoder-Text-Decoder model has been trained/fine-tuned, it can be saved/loaded just like any
other models (see the examples for more information).`),Vi.forEach(o),os=l($),Ke=a($,"P",{});var yn=s(Ke);ts=n(yn,"This model inherits from "),Ro=a(yn,"A",{href:!0});var $i=s(Ro);ns=n($i,"TFPreTrainedModel"),$i.forEach(o),rs=n(yn,`. Check the superclass documentation for the generic methods the
library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
etc.)`),yn.forEach(o),as=l($),Qe=a($,"P",{});var En=s(Qe);ss=n(En,"This model is also a "),Xe=a(En,"A",{href:!0,rel:!0});var qi=s(Xe);ds=n(qi,"tf.keras.Model"),qi.forEach(o),is=n(En,` subclass. Use it
as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
behavior.`),En.forEach(o),cs=l($),U=a($,"P",{});var To=s(U);Bo=a(To,"A",{href:!0});var zi=s(Bo);ls=n(zi,"TFVisionEncoderDecoderModel"),zi.forEach(o),ps=n(To,` is a generic model class that will be instantiated as a transformer architecture
with one of the base vision model classes of the library as encoder and another one of the base model classes as
decoder when created with the `),Vt=a(To,"CODE",{});var Pi=s(Vt);hs=n(Pi,"from_pretrained()"),Pi.forEach(o),ms=n(To,`class method for the encoder and
`),$t=a(To,"CODE",{});var Fi=s($t);fs=n(Fi,"from_pretrained()"),Fi.forEach(o),us=n(To,"class method for the decoder."),To.forEach(o),gs=l($),I=a($,"DIV",{class:!0});var J=s(I);g(eo.$$.fragment,J),_s=l(J),ie=a(J,"P",{});var ot=s(ie);vs=n(ot,"The "),Go=a(ot,"A",{href:!0});var Ci=s(Go);bs=n(Ci,"TFVisionEncoderDecoderModel"),Ci.forEach(o),Ts=n(ot," forward method, overrides the "),qt=a(ot,"CODE",{});var Ai=s(qt);ws=n(Ai,"__call__"),Ai.forEach(o),ys=n(ot," special method."),ot.forEach(o),Es=l(J),g(Te.$$.fragment,J),ks=l(J),zt=a(J,"P",{});var Si=s(zt);xs=n(Si,"Examples:"),Si.forEach(o),js=l(J),g(oo.$$.fragment,J),J.forEach(o),Ds=l($),O=a($,"DIV",{class:!0});var Ee=s(O);g(to.$$.fragment,Ee),Ms=l(Ee),Pt=a(Ee,"P",{});var Ii=s(Pt);Vs=n(Ii,`Instantiate an encoder and a decoder from one or two base classes of the library from pretrained model
checkpoints.`),Ii.forEach(o),$s=l(Ee),Ft=a(Ee,"P",{});var Li=s(Ft);qs=n(Li,"Example:"),Li.forEach(o),zs=l(Ee),g(no.$$.fragment,Ee),Ee.forEach(o),$.forEach(o),an=l(d),ce=a(d,"H2",{class:!0});var kn=s(ce);we=a(kn,"A",{id:!0,class:!0,href:!0});var Ni=s(we);Ct=a(Ni,"SPAN",{});var Oi=s(Ct);g(ro.$$.fragment,Oi),Oi.forEach(o),Ni.forEach(o),Ps=l(kn),At=a(kn,"SPAN",{});var Ri=s(At);Fs=n(Ri,"FlaxVisionEncoderDecoderModel"),Ri.forEach(o),kn.forEach(o),sn=l(d),x=a(d,"DIV",{class:!0});var q=s(x);g(ao.$$.fragment,q),Cs=l(q),le=a(q,"P",{});var tt=s(le);As=n(tt,`This class can be used to initialize an image-to-text-sequence model with any pretrained vision autoencoding model
as the encoder and any pretrained text autoregressive model as the decoder. The encoder is loaded via
`),St=a(tt,"CODE",{});var Bi=s(St);Ss=n(Bi,"from_pretrained()"),Bi.forEach(o),Is=n(tt,"function and the decoder is loaded via "),It=a(tt,"CODE",{});var Gi=s(It);Ls=n(Gi,"from_pretrained()"),Gi.forEach(o),Ns=n(tt,`function. Cross-attention layers are automatically added to the decoder and should be fine-tuned on a downstream
generative task, like image captioning.`),tt.forEach(o),Os=l(q),so=a(q,"P",{});var xn=s(so);Rs=n(xn,`The effectiveness of initializing sequence-to-sequence models with pretrained checkpoints for sequence generation
tasks was shown in `),io=a(xn,"A",{href:!0,rel:!0});var Wi=s(io);Bs=n(Wi,`Leveraging Pre-trained Checkpoints for Sequence Generation
Tasks`),Wi.forEach(o),Gs=n(xn,` by Sascha Rothe, Shashi Narayan, Aliaksei Severyn. Michael Matena, Yanqi
Zhou, Wei Li, Peter J. Liu.`),xn.forEach(o),Ws=l(q),co=a(q,"P",{});var jn=s(co);Us=n(jn,"Additionally, in "),lo=a(jn,"A",{href:!0,rel:!0});var Ui=s(lo);Hs=n(Ui,`TrOCR: Transformer-based Optical Character Recognition with Pre-trained
Models`),Ui.forEach(o),Zs=n(jn,` it is shown how leveraging large pretrained vision models for optical
character recognition (OCR) yields a significant performance improvement.`),jn.forEach(o),Ys=l(q),Lt=a(q,"P",{});var Hi=s(Lt);Js=n(Hi,`After such a Vision-Encoder-Text-Decoder model has been trained/fine-tuned, it can be saved/loaded just like any
other models (see the examples for more information).`),Hi.forEach(o),Ks=l(q),po=a(q,"P",{});var Dn=s(po);Qs=n(Dn,"This model inherits from "),Wo=a(Dn,"A",{href:!0});var Zi=s(Wo);Xs=n(Zi,"FlaxPreTrainedModel"),Zi.forEach(o),ed=n(Dn,`. Check the superclass documentation for the generic methods the
library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
etc.)`),Dn.forEach(o),od=l(q),ho=a(q,"P",{});var Mn=s(ho);td=n(Mn,`This model is also a Flax Linen
`),mo=a(Mn,"A",{href:!0,rel:!0});var Yi=s(mo);nd=n(Yi,"flax.nn.Module"),Yi.forEach(o),rd=n(Mn,` subclass. Use it as a
regular Flax Module and refer to the Flax documentation for all matter related to general usage and behavior.`),Mn.forEach(o),ad=l(q),H=a(q,"P",{});var wo=s(H);Uo=a(wo,"A",{href:!0});var Ji=s(Uo);sd=n(Ji,"FlaxVisionEncoderDecoderModel"),Ji.forEach(o),dd=n(wo,` is a generic model class that will be instantiated as a transformer architecture
with the module (flax.nn.Module) of one of the base vision model classes of the library as encoder module and
another one as decoder module when created with the :meth`),Nt=a(wo,"EM",{});var Ki=s(Nt);id=n(Ki,"~transformers.FlaxAutoModel.from_pretrained"),Ki.forEach(o),cd=n(wo,` class method
for the encoder and :meth`),Ot=a(wo,"EM",{});var Qi=s(Ot);ld=n(Qi,"~transformers.FlaxAutoModelForCausalLM.from_pretrained"),Qi.forEach(o),pd=n(wo," class method for the decoder."),wo.forEach(o),hd=l(q),L=a(q,"DIV",{class:!0});var K=s(L);g(fo.$$.fragment,K),md=l(K),pe=a(K,"P",{});var nt=s(pe);fd=n(nt,"The "),Ho=a(nt,"A",{href:!0});var Xi=s(Ho);ud=n(Xi,"FlaxVisionEncoderDecoderModel"),Xi.forEach(o),gd=n(nt," forward method, overrides the "),Rt=a(nt,"CODE",{});var ec=s(Rt);_d=n(ec,"__call__"),ec.forEach(o),vd=n(nt," special method."),nt.forEach(o),bd=l(K),g(ye.$$.fragment,K),Td=l(K),Bt=a(K,"P",{});var oc=s(Bt);wd=n(oc,"Examples:"),oc.forEach(o),yd=l(K),g(uo.$$.fragment,K),K.forEach(o),Ed=l(q),R=a(q,"DIV",{class:!0});var ke=s(R);g(go.$$.fragment,ke),kd=l(ke),Gt=a(ke,"P",{});var tc=s(Gt);xd=n(tc,`Instantiate an encoder and a decoder from one or two base classes of the library from pretrained model
checkpoints.`),tc.forEach(o),jd=l(ke),Wt=a(ke,"P",{});var nc=s(Wt);Dd=n(nc,"Example:"),nc.forEach(o),Md=l(ke),g(_o.$$.fragment,ke),ke.forEach(o),q.forEach(o),this.h()},h(){i(p,"name","hf:doc:metadata"),i(p,"content",JSON.stringify(hc)),i(M,"id","vision-encoder-decoder-models"),i(M,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(M,"href","#vision-encoder-decoder-models"),i(m,"class","relative group"),i(Eo,"href","/docs/transformers/pr_15839/en/model_doc/vision-encoder-decoder#transformers.VisionEncoderDecoderModel"),i(ko,"href","vit"),i(xo,"href","beit"),i(jo,"href","deit"),i(Do,"href","swin"),i(Mo,"href","roberta"),i(Vo,"href","gpt2"),i($o,"href","bert"),i(qo,"href","distilbert"),i(xe,"href","https://arxiv.org/abs/2109.10282"),i(xe,"rel","nofollow"),i(zo,"href","/docs/transformers/pr_15839/en/model_doc/vision-encoder-decoder#transformers.VisionEncoderDecoderModel"),i(Po,"href","trocr"),i(me,"id","transformers.VisionEncoderDecoderConfig"),i(me,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(me,"href","#transformers.VisionEncoderDecoderConfig"),i(X,"class","relative group"),i(Fo,"href","/docs/transformers/pr_15839/en/model_doc/vision-encoder-decoder#transformers.VisionEncoderDecoderConfig"),i(Co,"href","/docs/transformers/pr_15839/en/model_doc/vision-encoder-decoder#transformers.VisionEncoderDecoderModel"),i(Ao,"href","/docs/transformers/pr_15839/en/main_classes/configuration#transformers.PretrainedConfig"),i(So,"href","/docs/transformers/pr_15839/en/main_classes/configuration#transformers.PretrainedConfig"),i(Io,"href","/docs/transformers/pr_15839/en/model_doc/vision-encoder-decoder#transformers.VisionEncoderDecoderConfig"),i(ue,"class","docstring"),i(ge,"class","docstring"),i(z,"class","docstring"),i(_e,"id","transformers.VisionEncoderDecoderModel"),i(_e,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(_e,"href","#transformers.VisionEncoderDecoderModel"),i(te,"class","relative group"),i(Ce,"href","https://arxiv.org/abs/1907.12461"),i(Ce,"rel","nofollow"),i(Se,"href","https://arxiv.org/abs/2109.10282"),i(Se,"rel","nofollow"),i(Lo,"href","/docs/transformers/pr_15839/en/main_classes/model#transformers.PreTrainedModel"),i(Ne,"href","https://pytorch.org/docs/stable/nn.html#torch.nn.Module"),i(Ne,"rel","nofollow"),i(No,"href","/docs/transformers/pr_15839/en/model_doc/vision-encoder-decoder#transformers.VisionEncoderDecoderModel"),i(Oo,"href","/docs/transformers/pr_15839/en/model_doc/vision-encoder-decoder#transformers.VisionEncoderDecoderModel"),i(A,"class","docstring"),i(S,"class","docstring"),i(E,"class","docstring"),i(be,"id","transformers.TFVisionEncoderDecoderModel"),i(be,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(be,"href","#transformers.TFVisionEncoderDecoderModel"),i(se,"class","relative group"),i(Ze,"href","https://arxiv.org/abs/1907.12461"),i(Ze,"rel","nofollow"),i(Je,"href","https://arxiv.org/abs/2109.10282"),i(Je,"rel","nofollow"),i(Ro,"href","/docs/transformers/pr_15839/en/main_classes/model#transformers.TFPreTrainedModel"),i(Xe,"href","https://www.tensorflow.org/api_docs/python/tf/keras/Model"),i(Xe,"rel","nofollow"),i(Bo,"href","/docs/transformers/pr_15839/en/model_doc/vision-encoder-decoder#transformers.TFVisionEncoderDecoderModel"),i(Go,"href","/docs/transformers/pr_15839/en/model_doc/vision-encoder-decoder#transformers.TFVisionEncoderDecoderModel"),i(I,"class","docstring"),i(O,"class","docstring"),i(k,"class","docstring"),i(we,"id","transformers.FlaxVisionEncoderDecoderModel"),i(we,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),i(we,"href","#transformers.FlaxVisionEncoderDecoderModel"),i(ce,"class","relative group"),i(io,"href","https://arxiv.org/abs/1907.12461"),i(io,"rel","nofollow"),i(lo,"href","https://arxiv.org/abs/2109.10282"),i(lo,"rel","nofollow"),i(Wo,"href","/docs/transformers/pr_15839/en/main_classes/model#transformers.FlaxPreTrainedModel"),i(mo,"href","https://flax.readthedocs.io/en/latest/_autosummary/flax.nn.module.html"),i(mo,"rel","nofollow"),i(Uo,"href","/docs/transformers/pr_15839/en/model_doc/vision-encoder-decoder#transformers.FlaxVisionEncoderDecoderModel"),i(Ho,"href","/docs/transformers/pr_15839/en/model_doc/vision-encoder-decoder#transformers.FlaxVisionEncoderDecoderModel"),i(L,"class","docstring"),i(R,"class","docstring"),i(x,"class","docstring")},m(d,h){e(document.head,p),w(d,P,h),w(d,m,h),e(m,M),e(M,F),_(y,F,null),e(m,j),e(m,C),e(C,Vn),w(d,Jt,h),w(d,f,h),e(f,$n),e(f,Eo),e(Eo,qn),e(f,zn),e(f,rt),e(rt,Pn),e(f,Fn),e(f,ko),e(ko,Cn),e(f,An),e(f,xo),e(xo,Sn),e(f,In),e(f,jo),e(jo,Ln),e(f,Nn),e(f,Do),e(Do,On),e(f,Rn),e(f,at),e(at,Bn),e(f,Gn),e(f,Mo),e(Mo,Wn),e(f,Un),e(f,Vo),e(Vo,Hn),e(f,Zn),e(f,$o),e($o,Yn),e(f,Jn),e(f,qo),e(qo,Kn),e(f,Qn),w(d,Kt,h),w(d,he,h),e(he,Xn),e(he,xe),e(xe,er),e(he,or),w(d,Qt,h),w(d,G,h),e(G,tr),e(G,zo),e(zo,nr),e(G,rr),e(G,Po),e(Po,ar),e(G,sr),w(d,Xt,h),w(d,X,h),e(X,me),e(me,st),_(je,st,null),e(X,dr),e(X,dt),e(dt,ir),w(d,en,h),w(d,z,h),_(De,z,null),e(z,cr),e(z,fe),e(fe,Fo),e(Fo,lr),e(fe,pr),e(fe,Co),e(Co,hr),e(fe,mr),e(z,fr),e(z,ee),e(ee,ur),e(ee,Ao),e(Ao,gr),e(ee,_r),e(ee,So),e(So,vr),e(ee,br),e(z,Tr),e(z,it),e(it,wr),e(z,yr),_(Me,z,null),e(z,Er),e(z,ue),_(Ve,ue,null),e(ue,kr),e(ue,$e),e($e,xr),e($e,Io),e(Io,jr),e($e,Dr),e(z,Mr),e(z,ge),_(qe,ge,null),e(ge,Vr),e(ge,oe),e(oe,$r),e(oe,ct),e(ct,qr),e(oe,zr),e(oe,lt),e(lt,Pr),e(oe,Fr),w(d,on,h),w(d,te,h),e(te,_e),e(_e,pt),_(ze,pt,null),e(te,Cr),e(te,ht),e(ht,Ar),w(d,tn,h),w(d,E,h),_(Pe,E,null),e(E,Sr),e(E,ne),e(ne,Ir),e(ne,mt),e(mt,Lr),e(ne,Nr),e(ne,ft),e(ft,Or),e(ne,Rr),e(E,Br),e(E,Fe),e(Fe,Gr),e(Fe,Ce),e(Ce,Wr),e(Fe,Ur),e(E,Hr),e(E,Ae),e(Ae,Zr),e(Ae,Se),e(Se,Yr),e(Ae,Jr),e(E,Kr),e(E,ut),e(ut,Qr),e(E,Xr),e(E,Ie),e(Ie,ea),e(Ie,Lo),e(Lo,oa),e(Ie,ta),e(E,na),e(E,Le),e(Le,ra),e(Le,Ne),e(Ne,aa),e(Le,sa),e(E,da),e(E,W),e(W,No),e(No,ia),e(W,ca),e(W,gt),e(gt,la),e(W,pa),e(W,_t),e(_t,ha),e(W,ma),e(E,fa),e(E,A),_(Oe,A,null),e(A,ua),e(A,re),e(re,ga),e(re,Oo),e(Oo,_a),e(re,va),e(re,vt),e(vt,ba),e(re,Ta),e(A,wa),_(ve,A,null),e(A,ya),e(A,bt),e(bt,Ea),e(A,ka),_(Re,A,null),e(E,xa),e(E,S),_(Be,S,null),e(S,ja),e(S,Tt),e(Tt,Da),e(S,Ma),e(S,ae),e(ae,Va),e(ae,wt),e(wt,$a),e(ae,qa),e(ae,yt),e(yt,za),e(ae,Pa),e(S,Fa),e(S,Et),e(Et,Ca),e(S,Aa),_(Ge,S,null),w(d,nn,h),w(d,se,h),e(se,be),e(be,kt),_(We,kt,null),e(se,Sa),e(se,xt),e(xt,Ia),w(d,rn,h),w(d,k,h),_(Ue,k,null),e(k,La),e(k,de),e(de,Na),e(de,jt),e(jt,Oa),e(de,Ra),e(de,Dt),e(Dt,Ba),e(de,Ga),e(k,Wa),e(k,He),e(He,Ua),e(He,Ze),e(Ze,Ha),e(He,Za),e(k,Ya),e(k,Ye),e(Ye,Ja),e(Ye,Je),e(Je,Ka),e(Ye,Qa),e(k,Xa),e(k,Mt),e(Mt,es),e(k,os),e(k,Ke),e(Ke,ts),e(Ke,Ro),e(Ro,ns),e(Ke,rs),e(k,as),e(k,Qe),e(Qe,ss),e(Qe,Xe),e(Xe,ds),e(Qe,is),e(k,cs),e(k,U),e(U,Bo),e(Bo,ls),e(U,ps),e(U,Vt),e(Vt,hs),e(U,ms),e(U,$t),e($t,fs),e(U,us),e(k,gs),e(k,I),_(eo,I,null),e(I,_s),e(I,ie),e(ie,vs),e(ie,Go),e(Go,bs),e(ie,Ts),e(ie,qt),e(qt,ws),e(ie,ys),e(I,Es),_(Te,I,null),e(I,ks),e(I,zt),e(zt,xs),e(I,js),_(oo,I,null),e(k,Ds),e(k,O),_(to,O,null),e(O,Ms),e(O,Pt),e(Pt,Vs),e(O,$s),e(O,Ft),e(Ft,qs),e(O,zs),_(no,O,null),w(d,an,h),w(d,ce,h),e(ce,we),e(we,Ct),_(ro,Ct,null),e(ce,Ps),e(ce,At),e(At,Fs),w(d,sn,h),w(d,x,h),_(ao,x,null),e(x,Cs),e(x,le),e(le,As),e(le,St),e(St,Ss),e(le,Is),e(le,It),e(It,Ls),e(le,Ns),e(x,Os),e(x,so),e(so,Rs),e(so,io),e(io,Bs),e(so,Gs),e(x,Ws),e(x,co),e(co,Us),e(co,lo),e(lo,Hs),e(co,Zs),e(x,Ys),e(x,Lt),e(Lt,Js),e(x,Ks),e(x,po),e(po,Qs),e(po,Wo),e(Wo,Xs),e(po,ed),e(x,od),e(x,ho),e(ho,td),e(ho,mo),e(mo,nd),e(ho,rd),e(x,ad),e(x,H),e(H,Uo),e(Uo,sd),e(H,dd),e(H,Nt),e(Nt,id),e(H,cd),e(H,Ot),e(Ot,ld),e(H,pd),e(x,hd),e(x,L),_(fo,L,null),e(L,md),e(L,pe),e(pe,fd),e(pe,Ho),e(Ho,ud),e(pe,gd),e(pe,Rt),e(Rt,_d),e(pe,vd),e(L,bd),_(ye,L,null),e(L,Td),e(L,Bt),e(Bt,wd),e(L,yd),_(uo,L,null),e(x,Ed),e(x,R),_(go,R,null),e(R,kd),e(R,Gt),e(Gt,xd),e(R,jd),e(R,Wt),e(Wt,Dd),e(R,Md),_(_o,R,null),dn=!0},p(d,[h]){const vo={};h&2&&(vo.$$scope={dirty:h,ctx:d}),ve.$set(vo);const Ut={};h&2&&(Ut.$$scope={dirty:h,ctx:d}),Te.$set(Ut);const Ht={};h&2&&(Ht.$$scope={dirty:h,ctx:d}),ye.$set(Ht)},i(d){dn||(v(y.$$.fragment,d),v(je.$$.fragment,d),v(De.$$.fragment,d),v(Me.$$.fragment,d),v(Ve.$$.fragment,d),v(qe.$$.fragment,d),v(ze.$$.fragment,d),v(Pe.$$.fragment,d),v(Oe.$$.fragment,d),v(ve.$$.fragment,d),v(Re.$$.fragment,d),v(Be.$$.fragment,d),v(Ge.$$.fragment,d),v(We.$$.fragment,d),v(Ue.$$.fragment,d),v(eo.$$.fragment,d),v(Te.$$.fragment,d),v(oo.$$.fragment,d),v(to.$$.fragment,d),v(no.$$.fragment,d),v(ro.$$.fragment,d),v(ao.$$.fragment,d),v(fo.$$.fragment,d),v(ye.$$.fragment,d),v(uo.$$.fragment,d),v(go.$$.fragment,d),v(_o.$$.fragment,d),dn=!0)},o(d){b(y.$$.fragment,d),b(je.$$.fragment,d),b(De.$$.fragment,d),b(Me.$$.fragment,d),b(Ve.$$.fragment,d),b(qe.$$.fragment,d),b(ze.$$.fragment,d),b(Pe.$$.fragment,d),b(Oe.$$.fragment,d),b(ve.$$.fragment,d),b(Re.$$.fragment,d),b(Be.$$.fragment,d),b(Ge.$$.fragment,d),b(We.$$.fragment,d),b(Ue.$$.fragment,d),b(eo.$$.fragment,d),b(Te.$$.fragment,d),b(oo.$$.fragment,d),b(to.$$.fragment,d),b(no.$$.fragment,d),b(ro.$$.fragment,d),b(ao.$$.fragment,d),b(fo.$$.fragment,d),b(ye.$$.fragment,d),b(uo.$$.fragment,d),b(go.$$.fragment,d),b(_o.$$.fragment,d),dn=!1},d(d){o(p),d&&o(P),d&&o(m),T(y),d&&o(Jt),d&&o(f),d&&o(Kt),d&&o(he),d&&o(Qt),d&&o(G),d&&o(Xt),d&&o(X),T(je),d&&o(en),d&&o(z),T(De),T(Me),T(Ve),T(qe),d&&o(on),d&&o(te),T(ze),d&&o(tn),d&&o(E),T(Pe),T(Oe),T(ve),T(Re),T(Be),T(Ge),d&&o(nn),d&&o(se),T(We),d&&o(rn),d&&o(k),T(Ue),T(eo),T(Te),T(oo),T(to),T(no),d&&o(an),d&&o(ce),T(ro),d&&o(sn),d&&o(x),T(ao),T(fo),T(ye),T(uo),T(go),T(_o)}}}const hc={local:"vision-encoder-decoder-models",sections:[{local:"transformers.VisionEncoderDecoderConfig",title:"VisionEncoderDecoderConfig"},{local:"transformers.VisionEncoderDecoderModel",title:"VisionEncoderDecoderModel"},{local:"transformers.TFVisionEncoderDecoderModel",title:"TFVisionEncoderDecoderModel"},{local:"transformers.FlaxVisionEncoderDecoderModel",title:"FlaxVisionEncoderDecoderModel"}],title:"Vision Encoder Decoder Models"};function mc(Q,p,P){let{fw:m}=p;return Q.$$set=M=>{"fw"in M&&P(0,m=M.fw)},[m]}class Tc extends rc{constructor(p){super();ac(this,p,mc,pc,sc,{fw:0})}}export{Tc as default,hc as metadata};
