@cyyyyc123, I have implemented your idea adding a little since I don’t have the same number of samples in each hdf5. It looks like this:
class CT_dataset(data.Dataset):

  def __init__(self, path_patients):
      hdf5_list = [x for x in glob.glob(os.path.join(path_patients,'*.h5'))]#only h5 files
      print 'h5 list ',hdf5_list
      self.datasets = []
      self.datasets_gt=[]
      self.total_count = 0
      self.limits=[]
      for f in hdf5_list:
         h5_file = h5py.File(f, 'r')
         dataset = h5_file['data']
         dataset_gt = h5_file['label']
         self.datasets.append(dataset)
         self.datasets_gt.append(dataset_gt)
         self.limits.append(self.total_count)
         self.total_count += len(dataset)
         #print 'len ',len(dataset)
      #print self.limits   

  def __getitem__(self, index):
     
      dataset_index=-1
      #print 'index ',index
      for i in xrange(len(self.limits)-1,-1,-1):
        #print 'i ',i
        if index>=self.limits[i]:
          dataset_index=i
          break
      #print 'dataset_index ',dataset_index
      assert dataset_index>=0, 'negative chunk'

      in_dataset_index = index-self.limits[dataset_index]

      return self.datasets[dataset_index][in_dataset_index], self.datasets_gt[dataset_index][in_dataset_index]

  def __len__(self):
      return self.total_count 

It works fine except when I I use numworkers>=2 in the DataLoader.  Do you have any idea of what can be happening?