Currently have been experiencing this issue as well! I have doing an analysis with the decoder side, and I am pretty sure that the nn.TransformerDecoder is the root cause of this, Did you wanna try the following example?
#https://pytorch.org/docs/stable/generated/torch.nn.TransformerDecoder.html?highlight=nn%20transformerdecoder#torch.nn.TransformerDecoder
import torch
import torch.nn as nn
import math
pos_encoder = PositionalEncoding(d_model = 42,dropout=0.0)
torch.manual_seed(0)
memory = torch.rand(4, 1, 42) #src: (S, N, E) https://pytorch.org/docs/stable/generated/torch.nn.Transformer.html
tgt = pos_encoder(torch.rand(4, 1, 42)) #tgt: (T, N, E)
tgtmask = torch.nn.Transformer().generate_square_subsequent_mask(4).float()

decoder_layer = nn.TransformerDecoderLayer(d_model=42, nhead=7, dropout =0.0)
transformer_decoder = nn.TransformerDecoder(decoder_layer, num_layers=6)

criterion = nn.MSELoss()
optimizer = torch.optim.Adam(transformer_decoder.parameters(), lr = 0.05)

num_epochs = 1000

transformer_decoder.train()

for epoch in range(num_epochs):  
    
    out = transformer_decoder(tgt, memory, tgt_mask=tgtmask) 
    loss = criterion(out.permute(1,0,2), tgt.permute(1,0,2))
    print("predicted", out)
    print("target", tgt)
    # zeroes out the "old" parameter gradients and back prop
    optimizer.zero_grad()
    loss.backward()
    
    optimizer.step()

    print(f'Epoch {epoch + 1} | Loss for this epoch: {loss.item():.4f}')

@odats