Hey @mrshenli, Thanks for your reply.
I solved this issue already.  Basically,  there are two methods (three, plus your idea:D).
These ideas are mostly based on  DistributedSampler and DistributedDataParallel . Using the DistributedSampler, a subset of data can be loaded correctly in this process.
We can either use torch.multiprocessing to spawn a process manually to launch our training procedure,
def main():
   # Args defintion and loading
    os.environ['MASTER_ADDR'] = 'localhost'
    os.environ['MASTER_PORT'] = '8080'
    mp.spawn(train, nprocs=args.num_gpus, args=(args,))

def train(gpu, args):
    # Initialize the distributed package&group
    torch.distributed.init_process_group(backend='nccl', init_method='env://', world_size=args.world_size, rank=args.local_rank)
    torch.manual_seed(0)

    # Initial model
    model = RCNN(cfg)
    torch.cuda.set_device(gpu)
    model.cuda(gpu)
    
    # Wrap the model
    model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[gpu])

    # Initial dataset
    dataset = Dataset(cfg, is_train=True, split="train")

    # Kick-off iteration
    for epoch in range(cfg.TRAIN.MAX_EPOCH):
        loader = setup_dataloader(dataset,  is_distributed=True, epoch)
            
        for _, blobs in enumerate(loader):
             loss_dict = model.forward(blobs)

if __name__ == "__main__":
    main()

Passing args by using:
python3  tools/train_mp.py  --num_gpus 2

or we can use what torch already encapsulated:
class TrainingWrapper(object):
    def __init__(self, args):
        self.setup_logging()
        self.args = args

        # Initialize the distributed package&group
        self.num_gpus = torch.cuda.device_count()
        print("world_size:%d\local_rank:%d" % (args.num_gpus, args.local_rank))
        self.distributed = self.num_gpus > 1
        if self.distributed:
            torch.distributed.init_process_group(
                backend="nccl",
                init_method="env://",
                world_size=args.num_gpus,
                rank=args.local_rank
            )
        self.device = args.local_rank 
        
       # This line is very important!
        torch.cuda.set_device(self.device)

        # Initial model
        model = RCNN(cfg)

        # Distributed learning
        if self.distributed:
            model = torch.nn.parallel.DistributedDataParallel(
                model.cuda(self.device),
                device_ids=[self.args.local_rank],
                output_device=[self.args.local_rank],
                broadcast_buffers=False
            )
        else:
            model = torch.nn.DataParallel(model).cuda()

    def train(self):

        # Initial dataset
         dataset = Dataset(cfg, is_train=True, split="train")

        # Kick-off iteration
       for epoch in range(cfg.TRAIN.MAX_EPOCH):
            loader = setup_dataloader(dataset,  is_distributed=True, epoch)
            
            for _, blobs in enumerate(loader):
                 loss_dict = model.forward(blobs)

if  __name__ == "__main__":
    trainer = TrainingWrapper()
    trainer.train()

And passing following args for this above script:
python3 -m torch.distributed.launch --nproc_per_node=2 tools/train_ddp.py  --exp_id boa --config_file experiments/config.yaml --num_gpus 2

As so far, I can just use list of dict objects in order to feed the data in model.
Hopefully it helps somebody else somehow.