Transformers, Pytorch, Machine Learning, Deep Dives, Data Science.
torch.matmul(query, key.transpose(-2, -1)) / math.sqrt(d_k) # Attention mask here if attention_mask is not None: if attention_mask.dim() == 2: assert attention_mask.size() == (tgt_len, src_len) attention_mask = attention_mask.unsqueeze(0) logits = logits + attention_mask else: raise