The solution
def forward(self, x, y):
        """
        pass input through the encoder and reconstruct with decoder
        :param x: original input
        :param x: original target
        :return: x_recon : reconstructed input, z : latent representation
        """
        unpool_info = []
        real = torch.zeros(256).cuda()
        real[:128] = 0
        fake = torch.zeros(256).cuda()
        fake[128:] = 0
        mult = [real, fake]

        for m in self.encoder:
            if isinstance(m, nn.MaxPool2d):
                output_size = x.size()
                x, pool_idx = m(x)
                unpool_info.append({'output_size': output_size,
                                    'indices': pool_idx})
            else:
                x = m(x)
        z = x
        for i, y_i in enumerate(y):
            z[i] = z[i] * mult[y_i]

        for m in self.decoder:
            if isinstance(m, nn.MaxUnpool2d):
                x = m(x, **unpool_info.pop())
            else:
                x = m(x)
        x_recon = x
        return z, x_recon