Unity05:

but why don’t you just use an embedding layer


Thank you sir for your kind feedback.
I’m using this example, and my implementation is like :
class InputEmbeddingAndPositionalEncodingLayer(nn.Module):

    def __init__(self, vocab_size, max_len, d_model, dropout):
        super(InputEmbeddingAndPositionalEncodingLayer, self).__init__()
        self.vocab_size = vocab_size
        self.max_len = max_len
        self.d_model = d_model
        self.dropout = nn.Dropout(p=dropout)
        self.token_embedding = nn.Embedding(vocab_size, d_model)
        self.position_encoding = PositionalEncodingLayer(d_model=d_model, max_len=max_len)

    def forward(self, sequences):
        """
        :param Tensor[batch_size, seq_len] sequences
        :return Tensor[batch_size, seq_len, d_model]
        """
        token_embedded = self.token_embedding(sequences) # [batch_size, seq_len, d_model]
        position_encoded = self.position_encoding(sequences) # [batch_size, seq_len, d_model]
        return self.dropout(token_embedded) + position_encoded # [batch_size, seq_len, d_model]


shared_embeddings = InputEmbeddingAndPositionalEncodingLayer(
    vocab_size=len(TRG.vocab),
    max_len=MAX_LEN,
    d_model=D_MODEL,
    dropout=DROPOUT
)

transformer = Transformer(
    encoder=EncoderLayer(
        in_emb_pos_enc_layer=shared_embeddings,
        d_model=D_MODEL,
        n_heads=N_HEADS,
        hidden_size=HIDDEN_SIZE,
        dropout=DROPOUT,
        n_layers=N_LAYERS
    ),
    decoder=DecoderLayer(
        in_emb_pos_enc_layer=shared_embeddings,
        vocab_size=len(TRG.vocab),
        d_model=D_MODEL,
        n_heads=N_HEADS,
        hidden_size=HIDDEN_SIZE,
        dropout=DROPOUT,
        n_layers=N_LAYERS
    ),
    src_pad_index=TRG.vocab.stoi[TRG.pad_token],
    dest_pad_index=TRG.vocab.stoi[TRG.pad_token]
).to(DEVICE)

The result still not good as same as the implementation without shared embedding, could you give me an example?
One more thing, I used shared vocabulary size so no worries about the ‘train.csv’ and ‘valid.csv’ changes.