Thanks a lot for the reply! I should have read the docs more carefully, putting a weight for each sample makes sense.
I applied this to my MWE, however this trivial way for setting the weights gives a much higher weight for samples that have more than one class. Still iterating through the dataset with this samples gives counts of {'A': 44137.0, 'B': 37309.0, 'C': 20473.0} which is much less imbalanced than before.
Probably there is a better way for setting the weights.
import torch
import pandas as pd
import numpy as np
from torch.utils.data import Dataset
from sklearn.utils import shuffle
from torch.utils.data import DataLoader

len_ds = 60461
counts = {'A': 6775, 'B': 3609, 'C': 906}


def create_data(which_class: str):
    arr = np.zeros((1, len_ds))
    arr[:, :counts[which_class]] = 1
    return arr


data = {k: create_data(k) for k in counts}
data = np.concatenate((data['A'], data['B'], data['C']), axis=0)

label_df = pd.DataFrame(data).transpose()
label_df.columns = [*counts.keys()]
label_df = shuffle(label_df)
label_df.reset_index(inplace=True, drop=True)

y_train = label_df.values
class_sample_count = np.array([len(np.where(y_train == t)[0]) for t in np.unique(y_train)])

for idx, row in label_df.iterrows():
    label_df.at[idx, 'weights'] = (row['A']) * 1 / counts['A'] + (
        row['B']) * 1 / \
                                  counts['B'] + (row['C']) * 1 / counts['C'] + (
                                          row['A'] == row['B'] == row['C'] == 0) * 1 / (
                                          len(label_df) - counts['A'] - counts['B'] - counts['C'])

print(np.unique(label_df.weights, return_counts=True))
print(label_df[label_df == 1].count())


class toy_dataset(Dataset):
    def __getitem__(self, index):
        return torch.tensor(label_df.iloc[index].values)

    def __len__(self):
        return len(label_df)


dataset = toy_dataset()

weights = torch.DoubleTensor(label_df.weights.values)
print('weights: ', weights)
sampler = torch.utils.data.sampler.WeightedRandomSampler(weights, len(dataset), replacement=True)
trainloader = DataLoader(dataset, batch_size=50, sampler=sampler,
                         shuffle=False,
                         num_workers=0, pin_memory=False)

all_labels = torch.Tensor()
for labels in trainloader:
    all_labels = torch.cat((all_labels, labels.cpu()), 0)

print(all_labels.shape)

print(
    {label: all_labels[:, i].sum().item() for i, label in
     enumerate([*counts.keys()])})