This line just be used to save model:

  
      github.com
  
  
    ZPdesu/SEAN/blob/04c7536ff3fecd2d1a09c9ae046a1144636033a5/trainers/pix2pix_trainer.py#L23 1

updates the weights of the network while reporting losses
and the latest visuals to visualize the progress in training.
"""

def __init__(self, opt):
    self.opt = opt
    self.pix2pix_model = Pix2PixModel(opt)
    if len(opt.gpu_ids) > 0:
        self.pix2pix_model = DataParallelWithCallback(self.pix2pix_model,
                                                      device_ids=opt.gpu_ids)
        self.pix2pix_model_on_one_gpu = self.pix2pix_model.module
    else:
        self.pix2pix_model_on_one_gpu = self.pix2pix_model

    self.generated = None
    if opt.isTrain:
        self.optimizer_G, self.optimizer_D = \
            self.pix2pix_model_on_one_gpu.create_optimizers(opt)
        self.old_lr = opt.lr

def run_generator_one_step(self, data):



  
  
    
    
  
  


The program will crash in differenct epoches by different number’s gpus without error message.
But It is ok in one gpu.
I use pytorch launch function:
python -m torch.distributed.launch --nproc_per_node=$NGPUS train.py