Thank you Francisco.
But I didn’t do that. Below is part of my code
for batch_idx, batch_sample in enumerate(dataset_loader_train):
    inputs, label = batch_sample['seq'], batch_sample['label']
    if USE_CUDA:
        inputs, label = inputs.cuda(), label.cuda()
    label = label.view(inputs.size()[1])
    inputs = inputs.view(inputs.size()[1], 1, -1)
    inputs, label = Variable(inputs), Variable(label)
    optimizer.zero_grad()
    outputs = model(inputs)
    loss = criterion(outputs, label)
    loss.backward(retain_graph=True)
    optimizer.step()