I have somehow managed to achieve my goal using the following code:
data_dir = './data'  # the folder has multiple subfolders which contain the pt files
imgs_pt = []
labels_pt = []

i = 0
for root, dirs, files in os.walk(data_dir):
    for file in files:
        path = os.path.join(root, file)
        img_pt = torch.load(path)
        img_pt = img_pt.permute(2,0,1).unsqueeze(0)
        img_pt = img_pt.add(-mean_.cpu()).mul(1/std_.cpu())
        img_pt = img_pt.squeeze()
        label_pt = torch.tensor(i-1)
        labels_pt.append(label_pt)
        imgs_pt.append(img_pt)
    i += 1

labels_pt = torch.tensor(labels_pt)
print(labels_pt.shape)
print(labels_pt)
imgs_pt = torch.stack(imgs_pt)
print(imgs_pt.shape)

data = torch.utils.data.TensorDataset(imgs_pt, labels_pt)
data_loader = torch.utils.data.DataLoader(data,
                                          batch_size=1,
                                          shuffle=False,
                                          num_workers=0)

Just wondering if pytorch has a plan to embed such functionality for pt files as for images and numpy files, like ImageFolder and DataFolder