You could try to return the frames from each video as long as it’s possible and stack the different frames into the batch dimension.
This would change your actual batch size, but could work.
I’ve created a small example:
class VideoDataset(Dataset):
    def __init__(self):
        self.video1 = torch.randn(10, 3, 24, 24)
        self.video2 = torch.randn(20, 3, 24, 24)
        
    def __getitem__(self, index):
        if index < len(self.video1):
            x1 = self.video1[index]
        else:
            x1 = None

        if index < len(self.video2):
            x2 = self.video2[index]
        else:
            x2 = None
        
        x = torch.cat(([x.unsqueeze(0) for x in [x1, x2] if not x is None]))
        return x
        
    def __len__(self):
        return len(self.video2) # longest video

    
dataset = VideoDataset()

loader = DataLoader(
    dataset,
    batch_size=2,
    num_workers=2
)

for data in loader:
    print('Before ', data.shape)
    data = data.view(-1, 3, 24, 24)
    print('After ', data.shape)

Note that you would have to implement your frame reading operation instead of just slicing the tensor.