Hi Guys!!! I got a very important error!
DDP mode training normal, but when I resume the model , it got OOM. If I am not resume, training normal , the meory is enough.
So the problem is the resume part. But I am simple resume the state dict and I did nothing else. there are some operation do on the first GPU. I dont know why!!!
Here is my resume part of code:
last_weight = os.path.join(
        hyp.weight_path, "{}_last.pt".format(model_prefix))
    if os.path.exists(last_weight) and hyp.resume:
        if rank in [0, -1]:
            print('resume from last pth: ', last_weight)
        chkpt = torch.load(last_weight)
        # using local model to load state_dict, avoid module issue       
        local_model.load_state_dict(chkpt['model'])
        start_epoch = chkpt["epoch"] + 1
        if chkpt["optimizer"] is not None:
            optimizer.load_state_dict(chkpt["optimizer"])
            best_mAP = chkpt["best_mAP"]
        del chkpt
    else:
        if rank in [0, -1]:
            print('last pth not found or not resume, skip resume...')

As you can see, the local model is I move to the rank already!!
if hyp.train.sync_batch_norm and rank != -1:
        model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model)
        if rank == 0:
            logger.info('Using SyncBatchNorm()')
            logger.info('batch size: {} on single GPU, total batch size: {}'.format(
                bs, total_batch_size))
    if hyp.distributed:
        if rank == 0:
            logger.info(
                'Enable DDP mode, using all gpus. rank: {}'.format(rank))
        dist.init_process_group("nccl", rank=rank, world_size=world_size)
        local_model = model.to(rank)
        model = torch.nn.parallel.DistributedDataParallel(
            local_model, device_ids=[rank], output_device=rank)

        train_sampler = torch.utils.data.distributed.DistributedSampler(
            train_dataset, rank=rank, shuffle=True)
        # sampler_test = torch.utils.data.distributed.DistributedSampler(dataset_test)
        train_dataloader = DataLoader(
            train_dataset, sampler=train_sampler,  batch_size=bs, num_workers=hyp.train.num_workers, pin_memory=True)
    else:
        local_model = model.to(rank)
        train_dataloader = DataLoader(
            train_dataset, batch_size=bs, num_workers=hyp.train.num_workers, shuffle=True, pin_memory=True)

this is my distributed solving code.
Anyone knows why? I am really appreciated if anyone could help me out!!!