I’m wondering why multiplying the outputs of T5 by some scalar before inputting in the LM head :

  
      github.com
  
  
    huggingface/transformers/blob/1d2332861f5225aef17bd7e75abc670a72239081/src/transformers/modeling_tf_t5.py#L1147

        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
    )

    # insert decoder past at right place
    # to speed up decoding
    if use_cache is True:
        past = ((encoder_outputs, decoder_outputs[1]),)
        decoder_outputs = decoder_outputs[:1] + past + decoder_outputs[2:]

    sequence_output = decoder_outputs[0] * (self.model_dim ** -0.5)
    embed_tokens = self.get_output_embeddings()
    lm_logits = embed_tokens(sequence_output, mode="linear")
    decoder_outputs = (lm_logits,) + decoder_outputs[1:]

    return decoder_outputs + encoder_outputs

def prepare_inputs_for_generation(self, inputs, past, attention_mask, use_cache, **kwargs):
    assert past is not None, "past has to be defined for encoder_outputs"

    # first step



  
  
    
    
  
  


(Link to the original issue : https://github.com/huggingface/transformers/issues/5565 2)