I am building an improved-wasserstein style GAN, both generator and discriminator are RNN,
all is fine , but at the stage of  calculate the gradient_penalty, I got some error:

  File "/usr/local/lib/python3.5/dist-packages/torch/autograd/variable.py", line 156, in backward
    torch.autograd.backward(self, gradient, retain_graph, create_graph, retain_variables)
  File "/usr/local/lib/python3.5/dist-packages/torch/autograd/__init__.py", line 98, in backward
    variables, grad_variables, retain_graph)
RuntimeError: CudnnRNN is not differentiable twice


just like calc_gradient_penalty 3

def calc_gradient_penalty(netD, real_data, fake_data):
    alpha = torch.rand(BATCH_SIZE, 300 , 33)
    print(real_data.size())
    # alpha = alpha.expand(real_data.size())
    alpha = alpha.cuda()
    interpolates = alpha * real_data + ((1 - alpha) * fake_data)
    interpolates = Variable(interpolates, requires_grad=True)
    disc_interpolates = netD(interpolates)
    gradients = torch.autograd.grad(outputs=disc_interpolates, inputs=interpolates,
                              grad_outputs=torch.ones(disc_interpolates.size()).cuda(),
                              create_graph=True, retain_graph=True, only_inputs=True)[0]
    gradient_penalty = ((gradients.norm(2, dim=1) - 1) ** 2).mean() * LAMBDA
    return gradient_penalty


G and D:
class Generator(nn.Module):
    def __init__(self):
        super(Generator, self).__init__()
        self.lstm1 = nn.LSTM(
            input_size=128,
            hidden_size=33,
            num_layers=2,
            batch_first=True,
        )
        # self.out = nn.Linear(64, 33)
    
    def forward(self, x):
        r_out, (h_n, h_C) = self.lstm1(x, None)
        return r_out

class Discriminator(nn.Module):
    def __init__(self):
        super(Discriminator, self).__init__()
        self.lstm1 = nn.LSTM(
            input_size=33,
            hidden_size=64,
            num_layers=2,
            batch_first=True,
        )
        self.out = nn.Sequential(
            nn.Linear(64, 1),
            nn.Sigmoid()
            )


    def forward(self, x):
        r_out, (h_n, h_c) = self.lstm1(x, None)
        out = self.out(r_out[:, -1, :])
        return out

the input/output has the shape [batch_size, 300, 33], noise [batch_size, 300, 128]
Does any one know something?