Maybe I find the answer. After I see the source code of nn.DataParallel() (version 0.4.1).
    def __init__(self, module, device_ids=None, output_device=None, dim=0):
        super(DataParallel, self).__init__()

        if not torch.cuda.is_available():
            self.module = module
            self.device_ids = []
            return

        if device_ids is None:
            device_ids = list(range(torch.cuda.device_count()))
        if output_device is None:
            output_device = device_ids[0]
        self.dim = dim
        self.module = module
        self.device_ids = device_ids
        self.output_device = output_device

        _check_balance(self.device_ids)

        if len(self.device_ids) == 1:
            self.module.cuda(device_ids[0])

So there is no difference in performance whether we use multi-gpu setting or one-gpu setting, becuase multi-gpu setting use the one-gpu setting when len(device_ids) == 1.