Today I want to do distributed training in many nodes, however the memory decrease rapidly.

I execute the command 'dstat -c -m '.We can see the available memory decrease from 64GB to 177MB.
My batch_size is 1 and datasets include 4000 samples,every sample is about 16MB.I use the gloo as the backend.
data_loader code like this
class HDF5Dataset(data.Dataset):
    """Represents an abstract HDF5 dataset.
    
    Input params:
        file_path: Path to the folder containing the dataset (one or multiple HDF5 files).
        recursive: If True, searches for h5 files in subdirectories.
        transform: PyTorch transform to apply to every data instance (default=None).
        divide: use only 1//divide dataset
    """
    def __init__(self, file_path, recursive, transform=None, divide =1):
        super().__init__()
        self.data_info = []
        self.data_cache = {} #dict
        self.transform = transform
        self.length = 0
        self.divide = divide

        # Search for all h5 files
        p = Path(file_path)
        assert(p.is_dir())
        
        
        if recursive:
            files = sorted(p.glob('**/*.hdf5'))
        else:
            files = sorted(p.glob('*.hdf5'))
        if len(files) < 1:
            raise RuntimeError('No hdf5 datasets found')
        
        self.length = len(files) // divide
        self.data_info = files[0: self.length]
               
        #print("File Prepared !\n")
            
    def __getitem__(self, index):
        # get data
        y, x = self.get_data(index)
        
        if self.transform:
            x = self.transform(x)
        else:
            x = torch.from_numpy(x)
            x = torch.reshape(x, (1,128,128,128))
            x = x.type(torch.FloatTensor)

        # get label
        y = torch.from_numpy(y)
        y = y.type(torch.FloatTensor)
        return (x, y)

    def get_data(self, i):
        fp = self.data_info[i] #list - dict
        #print(fp)
        
        try:
            with h5py.File(fp,'r') as h5_file:
        
                label = h5_file.get('label')[()]            
                dataset = h5_file.get('dataset')[()]
            
                return (label, dataset)
        except IOError:
            print('Error:File {filepath} is broken.'.format(filepath=fp))
            i=i+1
            return self.get_data(i)
        
        
    def __len__(self):
        return self.length

I want to know why pytorch DDP training need more memory with the nodes expand. And how to reduce memory usage？