Uploading my solution:
from PIL import Image
from torchvision import datasets

def custom_loader(path):
    with open(path, 'rb') as f:
        img = Image.open(f)
        return img.convert('RGBA')
image_datasets = {x: datasets.ImageFolder(os.path.join(data_dir, x),data_transforms[x], 
                  loader=custom_loader)
                  for x in ['train', 'val']}