Yeah, you’re right. I want them to be connected to each other.
BTW, I tried what you mentioned and I am getting a similar error message:
RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.cuda.FloatTensor [60000, 1]], which is output 0 of TBackward, is at version 2; expected version 1 instead. Hint: the backtrace further above shows the operation that failed to compute its gradient. The variable in question was changed in there or anywhere later. Good luck!
Here I copy my discriminator class in case it is the reason for the issue:
class Discriminator(nn.Module):
def init(self, cfg):
super(Discriminator, self).init()
    self.seq_max = cfg.Dataset.Seq_Max
    self. num_joints = cfg.Dataset.Num_joint

    if cfg.Dataset.Num_dimension == 3:
        self. input_size = self.num_joints * 3
    else:
        self.input_size = self.num_joints * 2

    self.hidden_size = cfg.TRAIN.Hidden_size
    self.batch_size = cfg.TRAIN.Batch_Size
    self.layers = cfg.TRAIN.Num_Disc_Layers
    self.dnn_layers = cfg.TRAIN.Encoder_Dnn_Layers
    self.dropout = cfg.TRAIN.Encoder_Dropout
    self.bi = cfg.TRAIN.Bidirectional_Disc
    if self.dnn_layers > 0:
        for i in range(self.dnn_layers):
            self.add_module('dnn_' + str(i), nn.Linear(
                in_features=self.input_size if i == 0 else self.hidden_size,
                out_features=self.hidden_size
            ))
    gru_input_dim = self.input_size if self.dnn_layers == 0 else self.hidden_size
    self.rnn = nn.GRU(
        gru_input_dim,
        self.hidden_size,
        self.layers,
        dropout=self.dropout,
        bidirectional=self.bi,
        batch_first=True)
    self.relu = nn.LeakyReLU(0.2, inplace=False)
    self.fc = nn.Linear(self.hidden_size * self.seq_max * self.layers, 1)
    self.nonlinear = nn.Tanh()

def run_dnn(self, x):
    for i in range(self.dnn_layers):
        x = F.relu(getattr(self, 'dnn_'+str(i))(x))
    return x

def forward(self, inputs, hidden):
    output, embedding = self.rnn(inputs, hidden)
    output = self.relu(output).contiguous()
    output = output.view(output.size()[0], -1)
    output = self.fc(output)
    output = self.nonlinear(output)

    return output

The decoder and the encoder have similar architectures.