The whole traceback:
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-50-3dcd08b81ba1> in <module>()
      9             s2_in = s2_in.cuda()
     10 
---> 11         output, hidden_s1, hidden_s2 = model(s1_in, s2_in, hidden_s1, hidden_s2)
     12 #         hidden_s1 = repackage_hidden(hidden_s1)
     13 #         hidden_s2 = repackage_hidden(hidden_s2)

~/anaconda3/envs/py36/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    222         for hook in self._forward_pre_hooks.values():
    223             hook(self, input)
--> 224         result = self.forward(*input, **kwargs)
    225         for hook in self._forward_hooks.values():
     226             hook_result = hook(self, input, result)

<ipython-input-49-c51f58f6693e> in forward(self, s1_in, s2_in, hidden_s1, hidden_s2)
     27         # Embeddings shaped like: batch_size x padded seq length (40) x embedding_dimension (300)
     28 
---> 29         out_s1, hidden_s1 = self.lstm(emb_s1, hidden_s1)
      30         out_s2, hidden_s2 = self.lstm(emb_s2, hidden_s2)
     31 

~/anaconda3/envs/py36/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    222         for hook in self._forward_pre_hooks.values():
    223             hook(self, input)
 --> 224         result = self.forward(*input, **kwargs)
    225         for hook in self._forward_hooks.values():
     226             hook_result = hook(self, input, result)

~/anaconda3/envs/py36/lib/python3.6/site-packages/torch/nn/modules/rnn.py in forward(self, input, hx)
    160             flat_weight=flat_weight
    161         )
--> 162         output, hidden = func(input, self.all_weights, hx)
    163         if is_packed:
    164             output = PackedSequence(output, batch_sizes)

~/anaconda3/envs/py36/lib/python3.6/site-packages/torch/nn/_functions/rnn.py in forward(input, *fargs, **fkwargs)
    349         else:
    350             func = AutogradRNN(*args, **kwargs)
--> 351         return func(input, *fargs, **fkwargs)
    352 
    353     return forward

~/anaconda3/envs/py36/lib/python3.6/site-packages/torch/autograd/function.py in _do_forward(self, *input)
    282         self._nested_input = input
    283         flat_input = tuple(_iter_variables(input))
--> 284         flat_output = super(NestedIOFunction, self)._do_forward(*flat_input)
    285         nested_output = self._nested_output
    286         nested_variables = _unflatten(flat_output, self._nested_output)

~/anaconda3/envs/py36/lib/python3.6/site-packages/torch/autograd/function.py in forward(self, *args)
    304     def forward(self, *args):
    305         nested_tensors = _map_variable_tensor(self._nested_input)
--> 306         result = self.forward_extended(*nested_tensors)
    307         del self._nested_input
    308         self._nested_output = result

~/anaconda3/envs/py36/lib/python3.6/site-packages/torch/nn/_functions/rnn.py in forward_extended(self, input,     weight, hx)
     291             hy = tuple(h.new() for h in hx)
     292 
--> 293         cudnn.rnn.forward(self, input, hx, weight, output, hy)
     294 
     295         self.save_for_backward(input, hx, weight, output)

 ~/anaconda3/envs/py36/lib/python3.6/site-packages/torch/backends/cudnn/rnn.py in forward(fn, input, hx, weight,                          output, hy)
    264         if tuple(hx.size()) != hidden_size:
    265             raise RuntimeError('Expected hidden size {}, got {}'.format(
--> 266                 hidden_size, tuple(hx.size())))
    267         if cx is not None and tuple(cx.size()) != hidden_size:
    268             raise RuntimeError('Expected cell size {}, got {}'.format(

RuntimeError: Expected hidden size (1, 2, 8), got (4, 2, 8)

Basically the problem is that I can only have 1 lstm layer with cuda enabled. If I put lstm_layers to 1 it runs fine.