I made a function for this. It’s pretty specific and probably not optimal but works:
def pad_colors_random_choice(sequences):
    longest_colors = max([len(s) for s in sequences])
    
    out_tensor = torch.empty([len(sequences), longest_colors, 3], device=sequences[0].device)
    for i, colors in enumerate(sequences):
        n_colors = len(colors)
        diff = longest_colors - n_colors
        pad = colors[torch.randint(n_colors, [diff])]  
        out_tensor[i] = torch.cat([colors, pad])
        
    return out_tensor