I don’t know the details but after doing profiling I found that torch.cat is taking too many times. So I updated my code like this:
class Dataset():
    def __init__(self, df, path): 
        self.df = df
        self.path = path
        
    def __len__(self): 
        return len(self.df)
    def __getitem__(self, idxs):
        if isinstance(idxs, int):
            imgs = transform_image(self.path/self.df.iloc[idxs]["fname"])
            labels = create_code(self.df, idxs)
        else:
            sub_df = self.df.iloc[idxs]
            imgs = []
            labels = []
            for i in range(len(idxs)):
                e = sub_df.iloc[i]["fname"]
                imgs.append(transform_image(self.path/e))
                labels.append(create_code(sub_df, i))
            
        return torch.stack(imgs), torch.stack(labels)
            

Instead of torc.cat, I am appending the list and then transforming the list and stacking elements, now iteration over 1 epoch takes 11.2 s