Transformers, Pytorch, Machine Learning, Deep Dives, Data Science.
self).__init__() self.hidden_dim = hidden_dim self.num_heads = num_heads assert hidden_dim % num_heads == 0, "Hidden dim must be divisible by num heads" self.Wv = nn.Linear(hidden_dim, hidden_dim, bias=False) # the Value part self.Wk = nn.Linear(hidden_dim, hidden_dim, bias=False) # the Key part