Thanks! I tried your suggestion, but it only addresses my initial issue, which was fixed using net.model.init_hidden call, it didn’t help with the second issue - the batch not being split between GPUs.
My guess is it’s related to the tensors inside forward function not having first dimension as batch_size. I used batch_first=True parameter in GRU initialization, so that I could use inputs shaped as (batch, seq_len). However, I’m not sure what to do with hidden tensor, which is (num_layers, batch, hidden_size).
Now, when I use RNN class like this:
class RNN(nn.Module):
    def __init__(self, input_size, hidden_size, output_size, n_layers):
        super(RNN, self).__init__()
        self.input_size = input_size
        self.hidden_size = hidden_size
        self.output_size = output_size
        self.n_layers = n_layers 
        
        self.encoder = nn.Embedding(input_size, hidden_size)
        self.GRU = nn.GRU(hidden_size, hidden_size, n_layers, batch_first=True) 
        self.decoder = nn.Linear(hidden_size, output_size)
        
        
    def forward(self, input, batch_size):
        self.init_hidden(batch_size)
        input = self.encoder(input)
        output, self.hidden = self.GRU(input, self.hidden) 
        output = self.decoder(output) #batch_first
        return output
    
    def init_hidden(self, batch_size):
        self.hidden = Variable(torch.randn(self.n_layers, batch_size, self.hidden_size).cuda())

I get the following error (batch_size=64, hidden_size=256, 2 layers, 4 GPUs):
Traceback (most recent call last):
  File "char-rnn.py", line 232, in <module>
    output = net(c, batch_size)
  File "/root/miniconda2/lib/python2.7/site-packages/torch/nn/modules/module.py", line 206, in __call__
    result = self.forward(*input, **kwargs)
  File "/root/miniconda2/lib/python2.7/site-packages/torch/nn/parallel/data_parallel.py", line 61, in forward
    outputs = self.parallel_apply(replicas, inputs, kwargs)
  File "/root/miniconda2/lib/python2.7/site-packages/torch/nn/parallel/data_parallel.py", line 71, in parallel_apply
    return parallel_apply(replicas, inputs, kwargs)
  File "/root/miniconda2/lib/python2.7/site-packages/torch/nn/parallel/parallel_apply.py", line 45, in parallel_apply
    raise output
RuntimeError: Expected hidden size (2, 16L, 256), got (2L, 64L, 256L)