Gpt, Transformer, Pytorch, Multi Head Attention, Decoder.
class DecoderBlock(nn.Module): def __init__(self, d_model, num_heads, ff_hidden_layer, dropout): super(DecoderBlock, self).__init__() self.self_attention = nn.MultiheadAttention(d_model, num_heads, dropout=dropout) self.norm1 = nn.LayerNorm(d_model) self.dropout1 = nn.Dropout(dropout) self.linear1