I was able to do the changes I needed on the labels by exploiting the target_transform parameter.
class ImageNet1k:
    def __init__(self, path='/mnt/data/rajivratn/maiti/imagenet/imagenet-val'): 
        self.valdir = path
        self.data = None
    
    def make_dataset(self):
        normalize = transforms.Normalize(mean=[0.5, 0.5, 0.5], 
                                        std=[0.5, 0.5, 0.5])
        self.data = datasets.ImageFolder(self.valdir, transforms.Compose([
                transforms.Resize(256),
                transforms.CenterCrop(224),
                transforms.ToTensor(),
                normalize,
            ]))
        self.class_to_idx = self.data.class_to_idx 
        self.idx_to_class = {v: int(k) for k,v in self.class_to_idx.items()}
        return self.data

    def get_val_dataloader(self, batch_size, num_workers=0):
        if self.data is None:
             self.make_dataset() 
        val_loader = torch.utils.data.DataLoader(
            self.data,
            batch_size=batch_size, shuffle=False,
            num_workers=num_workers, pin_memory=False)
        
        return val_loader


if __name__ == '__main__':
    args = parseargs() 
    if args.gpu == -1:
        device = 'cpu'
    else:
        device = f'cuda:{args.gpu}' if torch.cuda.is_available() else 'cpu'
    model = vit_base_patch16_224(pretrained=True) 
    model = model.to(device)
    in1k = ImageNet1k() 
    val_dataloader = in1k.get_val_dataloader(batch_size=args.batch_size) 
    def target_transform(label):
        return in1k.idx_to_class[label] 
    
    in1k.data.target_transform = target_transform
    
    model.eval() 
    print(f'==> Training state of mode: {model.training}')
    total = 0
    correct = 0
    for batch_idx, (images, labels) in tqdm(enumerate(val_dataloader)):
            images, labels = images.to(device), labels.to(device) 
            y_preds = model(images) 
            y_preds = torch.argmax(y_preds, dim=1)
            # labels = torch.tensor([in1k.idx_to_class[label.item()] for label in labels]).to(device)
            correct += (y_preds == labels).sum()
            total += y_preds.shape[0]
    print(f'Current Accuracy: {correct / total:.2f}')