I actually have narrowed the issue down.
it seems to be happening when I tried to use more than 1 GPU.
def prepare_device(n_gpu_use):
    """
    setup GPU device if available, move model into configured device
    """
    n_gpu = torch.cuda.device_count()
    if n_gpu_use > 0 and n_gpu == 0:
        print("Warning: There\'s no GPU available on this machine,"
                            "training will be performed on CPU.")
        n_gpu_use = 0
    if n_gpu_use > n_gpu:
        print("Warning: The number of GPU\'s configured to use is {}, but only {} are available "
                            "on this machine.".format(n_gpu_use, n_gpu))
        n_gpu_use = n_gpu
    device = torch.device('cuda:0' if n_gpu_use > 0 else 'cpu')
    list_ids = list(range(n_gpu_use))
    return device, list_ids

self.device, self.device_ids = prepare_device(4)

self.discriminator['model'] = self.discriminator['model'].to(self.device)
if len(self.device_ids) > 1:
      self.discriminator['model'] = torch.nn.DataParallel(self.discriminator['model'], device_ids=self.device_ids)

and input data is not parallelized. (not even sure if it has to be…)
I just call .to(self.device) on those tensors