Transformers, Pytorch, Machine Learning, Deep Dives, Data Science.
self.mha = MultiHeadAttention(hidden_dim=n_dim, num_heads=n_heads) self.norm1 = nn.LayerNorm(n_dim) self.ff = PositionWiseFeedForward(n_dim, n_dim) self.norm2 = nn.LayerNorm(n_dim) self.dropout = nn.Dropout(dropout) def forward(self, x, src_padding_mask=None): assert x.ndim==3, "Expected input to