Here’s an example class for the STL10 data set. I’d be happy to add the indexes to the datasets and submit a pull request if that would be useful.
class STL10DataSet(Dataset):
"""
STL10 data set. Implemented to be able to get the index of the images
in a mini-batch so that predictions can be associated with their original
image data.

https://github.com/mttk/STL10/blob/master/stl10_input.py
"""

def __init__(self, root_dir, split='train', transform=None):
    """
    Args:
        root_dir (string): path where stl10_binary.tar.gz has been extracted.
        transform (callable, optional): Optional transform to be applied on an
            image.
    """
    self.root_dir = root_dir
    self.split = split
    self.transform = transform
    
    with open(os.path.join(self.root_dir, '{}_y.bin'.format(self.split)), 'rb') as f:
        self.labels = np.fromfile(f, dtype=np.uint8) - 1
        
    with open(os.path.join(self.root_dir, 'class_names.txt')) as f:
        self.classes = f.read().splitlines()
        
def __len__(self):
    return len(self.labels)

def __getitem__(self, idx):
    with open(os.path.join(self.root_dir, '{}_X.bin'.format(self.split)), 'rb') as f:
        c, h, w = 3, 96, 96
        size = c*h*w
        f.seek(idx*size)
        # read bytes from file
        image = np.fromfile(f, dtype=np.uint8, count=size)
        # reshape to C x H x W
        image = np.reshape(image, (c, h, w))
        # transpose to H x W x C for ToPILImage transform
        image = np.transpose(image, (2, 1, 0))
        
        if self.transform:
            image = self.transform(image)
            
        return image, self.labels[idx], idx