Oh sorry, I missed this requirement. So it seems you would like to disallow the “odd” mapping, such as (1, 2) and remove these pairs? If so, could you check if this would work:
class JointDataset(torch.utils.data.Dataset):
    def __init__(self):

        # Create random dataset and encode data via target
        self.mnist_targets_1 = torch.randint(0, 10, (100,))
        # remove invalid targets
        self.mnist_targets_1 = self.mnist_targets_1[self.mnist_targets_1%2==0]
        self.mnist_data_1 = self.mnist_targets_1 + 0.1
        

        self.mnist_targets_2 = torch.randint(0, 10, (100,))
        self.mnist_data_2 = self.mnist_targets_2 + 0.2
        self.mnist_target_idx_mapping = self.process_mnist_labels()

    def process_mnist_labels(self):
        numbers_dict = {0: [], 2: [], 4:[], 6:[], 8:[]}
        for i in range(len(self.mnist_targets_2)):
            mnist_target = self.mnist_targets_2[i].item()
            if mnist_target % 2 != 0:
                numbers_dict[(mnist_target-1)%10].append(i)
        return numbers_dict
        
        
    def __len__(self):
        return len(self.mnist_data_1)
        
    def __getitem__(self, index: int):
        """
        Args:
            index (int): Index
        Modality 1: even digits
        Modality 2: odd  digits
        """
        mnist_img_1, mnist_target_1 = self.mnist_data_1[index], int(self.mnist_targets_1[index])


        indices_list = self.mnist_target_idx_mapping[(mnist_target_1)]
        # Randomly pick an index from the indices list
        idx = random.choice(indices_list)

        mnist_img_2    = self.mnist_data_2[idx]
        mnist_target_2 = int(self.mnist_targets_2[idx])
        
        
        return mnist_img_1, mnist_img_2, mnist_target_1, mnist_target_2
    
dataset = JointDataset()
for data in dataset:
    print(data)

Output:
(tensor(8.1000), tensor(9.2000), 8, 9)
(tensor(4.1000), tensor(5.2000), 4, 5)
(tensor(8.1000), tensor(9.2000), 8, 9)
(tensor(8.1000), tensor(9.2000), 8, 9)
(tensor(0.1000), tensor(1.2000), 0, 1)
(tensor(8.1000), tensor(9.2000), 8, 9)
...