I change my implementation into a more simpler one :
class LPU(nn.Module):
    def __init__(self, Encoder_size, Hidden_size, Decoder_size):
        # simple autoencoder structure
        super(LPU, self).__init__()
        # ENC input : main sensory state (Encoder_size), the last hidden state
        # and last hidden state from a superior LPU
        self.encoder = nn.Linear((Encoder_size + 2*Hidden_size), Hidden_size)
        self.act_encoder = nn.Sigmoid()

        self.decoder = nn.Linear(Hidden_size, Decoder_size)
        self.act_decoder = nn.Sigmoid()

    def forward(self, Xt, last_Hidden, last_Hidden_sup):

        input_encoder = torch.cat((Xt, last_Hidden, last_Hidden_sup), 1)
        encoder_process = self.encoder(input_encoder)
        representation = self.act_decoder(encoder_process)

        decoder_process = self.decoder(representation)
        out_decoder = self.act_decoder(decoder_process)

        return out_decoder, representation