I did that and it fails on 6021-th index. Here is my simple custom dataset. The length of the dataframe is 6134. I tried removing the csv entry at 6021th index and trying again but the dataset fails at the same index again.
class FlowerBirds(Dataset):

    def __init__(self, reverse=True, mode='train', transform=None):
        ROOT = 'data_stuf/'
        if reverse:
            self.df = pd.read_csv(ROOT+mode+'_'+'bf.csv')
        else:
            self.df = pd.read_csv(ROOT+mode+'_'+'fb.csv')
        self.tfm = transform
        #print(len(self.df), 'DF LEN')
        #self.normalize = transforms.Normalize(mean=[0.491, 0.482, 0.447], std=[0.247, 0.243, 0.262]) 

    def __getitem__(self, index):
        #print(index, "INDEX")
        img = Image.open(self.df['PATH'].iloc[index])
        label = self.df['LABEL'].iloc[index]
        img = self.tfm(img)
        if img.shape[0] == 1:
            img = img.repeat(3, 1, 1)
        #img = self.normalize(img)
        label = torch.LongTensor(np.array([label])).item()
        return img, label

    def __len__(self):
        return len(self.df)