Hi,
So I have investigated my dataloader and it appears to be very slow. This causes the GPU utlization to be zero as not enough samples are fed into the GPU (i’m assuming). I have around 150k instances to process from a hdf5 dataset. Here is my dataset class:
class Features_Dataset(data.Dataset):
    def __init__(self, archive, phase):
        self.archive = archive
        self.phase = phase

    def __getitem__(self, index):
        with h5py.File(self.archive, 'r', libver='latest', swmr=True) as archive:
            datum = archive[str(self.phase) + '_all_arrays'][index]
            label = archive[str(self.phase) + '_labels'][index]
            path = archive[str(self.phase) +  '_img_paths'][index]
            return datum, label, path

    def __len__(self):
        with h5py.File(self.archive, 'r', libver='latest', swmr=True) as archive:
            datum = archive[str(self.phase) + '_all_arrays']
            return len(datum)


if __name__ == '__main__':
    train_dataset = Features_Dataset(archive= "featuresdata/train.hdf5", phase= 'train')
    trainloader = data.DataLoader(train_dataset, num_workers=1, batch_size=4)
    print(len(trainloader))
    for i, (data, label, path) in enumerate(trainloader):
        print(path)



Is there a way of speeding this up? I’m not really sure what else there is to do…