Your code works fine using dummy data:
class MyDataset(Dataset):
    def __init__(self):
        self.data_grain = torch.randn(100, 3, 64, 64)
        self.data_mask = torch.randn(100, 1, 64, 64)
        
    def __getitem__(self, index):
        x_grain = self.data_grain[index]
        x_mask = self.data_mask[index]
        x = torch.cat((x_grain, x_mask), dim=0)
        return x
    
    def __len__(self):
        return len(self.data_grain)

dataset = MyDataset()

dataloader = torch.utils.data.DataLoader(
    dataset, batch_size=64, shuffle=True, num_workers=0)

data = next(iter(dataloader))
print(data.shape)

Could you check the opt.batchSize again?
Also, could you print the shape of the output of dataset.data_grain[0] as well as dataset.data_mask[0]?