I don’t know what might be causing the issue as I’m able to overfit random data using your model and this code:
N = 64
data = torch.randn(N, 10, 1)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

model = Discriminator(hidden_size=32, target_seq_len=1, num_classes=5)
optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
loss_fn = nn.CrossEntropyLoss()
model.train()

train_batch = data
train_lbls = torch.randint(0, 5, (N,)).to(device)


for i in range(1000):    
    optimizer.zero_grad()
    lstm_hidden = model.init_lstm_memory(train_batch.shape[0]) 
    preds, lstm_hidden = model(train_batch, lstm_hidden)
    loss = loss_fn(preds, train_lbls)
    loss.backward()
    optimizer.step()
    _, preds = torch.max(preds, dim=1)
    print("epoch {}, loss {}, acc {}".format(
        i, loss.detach().item(), (preds == train_lbls).float().mean()))