Hello @Varg_Nord !
In my experiments, DataParallel works correctly with multi layer RNN. Example is provided below:
class RNNLM(nn.Module):

    def __init__(self, ntokens=100000, nx=300, nhid=600,
                 nlayers=3, dropout=0.5):
        super(Model, self).__init__()

        self.keep_prob = dropout
        self.ntokens = ntokens
        self.nx = nx
        self.nhid = nhid
        self.nlayers = nlayers

        self.dropout = nn.Dropout(p=self.keep_prob)

        self.embs = nn.Embedding(num_embeddings=self.ntokens,
                                 embedding_dim=self.nx,
                                 padding_idx=constants.PAD)

        self.rnn = nn.LSTM(input_size=self.nx,
                           hidden_size=self.nhid,
                           num_layers=self.nlayers,
                           batch_first=True,
                           dropout=self.keep_prob)

        self.linear = nn.Linear(in_features=self.nhid,
                                out_features=self.ntokens)

        self.init_weights()

    def init_weights(self):
        initrange = 0.1
        self.embs.weight.data.uniform_(-initrange, initrange)
        self.linear.bias.data.fill_(0)
        self.linear.weight.data.uniform_(-initrange, initrange)

    def forward(self, x, maxlen, conds, hidden=None):

        lengths = x.ne(constants.PAD).sum(dim=1).data.cpu().view((-1)).numpy()

        embs = self.embs(x)
        embs = pack(embs, lengths, batch_first=True)

        output, hidden = self.rnn(embs, hidden)
        output = unpack(output, batch_first=True)[0]
        output = self.dropout(output)

        padded_output = Variable(
            torch.zeros(output.size()[0], maxlen, output.size()[2])
        ).cuda()

        padded_output[:, :max(lengths), :] = output

        decoded = self.linear(
            padded_output.view(
                padded_output.size(0) * padded_output.size(1),
                padded_output.size(2)
            )
        )

        return decoded, hidden

net = torch.nn.DataParallel(Model(ntokens=nvocab, nx=300, nhid=300, nlayers=3, dropout=0.5).cuda())
Also, don’t forget to set env for CUDA_VISIBLE_DEVICES.
If you will have some error with this code then provide the error and your setup.