Why does the transformer tutorial have a multiplication by sqrt number of inputs? I know there is a division by sqrt(D) in the multiheaded self attention, but why is there one with the output of the encoder?
In particular:
src = self.encoder(src) * math.sqrt(self.ninp)

from: Language Modeling with nn.Transformer and TorchText — PyTorch Tutorials 1.9.0+cu102 documentation 3
Reference code:
import math

import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.nn import TransformerEncoder, TransformerEncoderLayer

class TransformerModel(nn.Module):

    def __init__(self, ntoken, ninp, nhead, nhid, nlayers, dropout=0.5):
        super(TransformerModel, self).__init__()
        self.model_type = 'Transformer'
        self.pos_encoder = PositionalEncoding(ninp, dropout)
        encoder_layers = TransformerEncoderLayer(ninp, nhead, nhid, dropout)
        self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers)
        self.encoder = nn.Embedding(ntoken, ninp)
        self.ninp = ninp
        self.decoder = nn.Linear(ninp, ntoken)

        self.init_weights()

    def generate_square_subsequent_mask(self, sz):
        mask = (torch.triu(torch.ones(sz, sz)) == 1).transpose(0, 1)
        mask = mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(mask == 1, float(0.0))
        return mask

    def init_weights(self):
        initrange = 0.1
        self.encoder.weight.data.uniform_(-initrange, initrange)
        self.decoder.bias.data.zero_()
        self.decoder.weight.data.uniform_(-initrange, initrange)

    def forward(self, src, src_mask):
        src = self.encoder(src) * math.sqrt(self.ninp)
        src = self.pos_encoder(src)
        output = self.transformer_encoder(src, src_mask)
        output = self.decoder(output)
        return output

or
self.embedding(tokens.long()) * math.sqrt(self.emb_size)`


  

      stats.stackexchange.com
  

  
      
    
  


  Why are the embeddings of tokens multiplied by $\sqrt D$ (note not divided by square root of D) in a transformer? 1



  machine-learning, neural-networks, natural-language, transformers



  asked by
  
  
    Charlie Parker
  
  on 10:27PM - 14 Jul 21 UTC


  

  
    
    
  

  



  
      

      reddit
  

  
    

r/learnmachinelearning - Why does the embedding of tokens to the transformer...

  2 votes and 0 comments so far on Reddit