github.com
  
  
    pytorch/pytorch/blob/master/torch/nn/parallel/parallel_apply.py#L22 8

    for result in results:
        if isinstance(result, Variable):
            return result
if isinstance(obj, dict):
    results = map(get_a_var, obj.items())
    for result in results:
        if isinstance(result, Variable):
            return result
return None




def parallel_apply(modules, inputs, kwargs_tup=None, devices=None):
assert len(modules) == len(inputs)
if kwargs_tup is not None:
    assert len(modules) == len(kwargs_tup)
else:
    kwargs_tup = ({},) * len(modules)
if devices is not None:
    assert len(modules) == len(devices)
else:
    devices = [None] * len(modules)



  
  
    
    
  
  


This seems to constrain input to variables or a list (or nested list)  of variables. I suppose that at least allowing dictionary of variables as input should increase flexibility (which indeed works for single GPU case). I created an issue for it.
If you think this bug should be resolved, I would be happy to send a pull request.