I think the problem is inside __getitem__ method. Remaining parts are working as they should.
This is the code after removing augmentations and transformations.
class SegDataset(data.Dataset):
	def __init__(self, root, split="train"):
		self.root=root
		self.split = split
		self.n_classes = 3
		self.files = collections.defaultdict(list)
		self.images_base = os.path.join(self.root, self.split)
		self.masks_base = os.path.join(self.root, 'masks' + '/' + self.split)
		for split in ['train', 'val', 'test']:
			file_list = recursive_glob(rootdir=self.root + '/' + split , suffix='.jpg')
			self.files[split] = file_list

	def __len__(self):
		return len(self.files[self.split])

	def __getitem__(self, index):
		img_path = self.files[self.split][index].rstrip()
		img_name = img_path.split(os.sep)[-1].split('.')[0]
		mask_name = img_name + 	'_mask.png'
		mask_path = self.masks_base + '/' + mask_name
		
		img = io.imread(img_path)
		mask = io.imread(mask_path)
		return img, mask

When I try to sample one datapoint from the dataset, it still runs out of RAM.
code used for sampling.
path = '\path\to_dataset\
dst = FootDataset(root=path, split='train')
data = dst[22]

The process dies after this code snippet runs for some time. Never seen this kind of behaviour in dtaloaders.