Depending on how much data there is, it might be simpler to just filter out data elements at inference time (with a moderate penalty on performance due to smaller batch sizes).
# sketch of idea, might need refinement depending on specific shape
def filter_data(x_batch, y_batch, skip_classes):
    for class in skip_classes:
        y_batch[y_batch==class] = -1
    mask = y_batch >= 0
    filtered_x_batch = x_batch[mask,:]
    filtered_y_batch = y_batch[mask]
    return filtered_x_batch, filtered_y_batch
...
# Test model performance for classes with less than 20 samples.

y_pred_list = []
y_true_list = []
with torch.no_grad():
    for x_batch, y_batch in tqdm(data_loader):
        x_batch, y_batch = x_batch.to(device), y_batch.to(device)
        x_batch, y_batch = filter_data(x_batch, y_batch, skip_classes)
        y_test_pred = model(x_batch)
        _, y_pred_tag = torch.max(y_test_pred, dim = 1)
        y_pred_list.append(y_pred_tag.cpu().numpy())
        y_true_list.append(y_batch.cpu().numpy())