I am using 1.3.1 in GPU server (TitanV * 2). And I try to run a single-GPU module on multiple GPU.
My model can run now. Thanks.
@albanD That is what I changed:
        decoder_in, s, w = decoder_initial(x.size(0))
        decoder_in = y[:, 0]
        decoder_in_1 = decoder_in.to('cuda:1')

        # 1.7. for each decoder timestep
        for j in range(y.size(1) - 1):  # for all sequences
            """
            decoder_in (Variable): [b]
            encoded (Variable): [b x seq x hid]
            input_out (np.array): [b x seq]
            s (Variable): [b x hid]
            """
            # 1.7.1.1st state - create [out]
            if j == 0:
                h_out, c_out, h_add = midlstm(None, None, y=decoder_in, order=j, encoded=encoded)
                if torch.cuda.is_available():
                    h_add = h_add.to('cuda:1')

And in midlstm layer:
    def forward(self, h_0, c_0, y, order, encoded):

        y[y >= self.vocab_size] = 1  # y is narray
        b = encoded.size(0)  # batch size
        seq = encoded.size(1)  # input sequence length
        hidden_size = self.hidden_size

        # I delete
        #  `y = torch.LongTensor(y) `
        #  and it works
        att = self.Emb(y).unsqueeze(1)
        # In __init__: self.Emb = nn.Embedding(vocab_size, seq_length)
        inputs = torch.bmm(att, encoded).squeeze()