Thanks for the reply!
I solved the problem using a custom Sampler class and changing the num_samples parameter:
class GroupSampler(Sampler):

    def __init__(self, dataset, samples_per_gpu=1):
        self.dataset = dataset
        self.samples_per_gpu = samples_per_gpu
        steps_per_epoch = 4
        self.num_samples = steps_per_epoch * self.samples_per_gpu

However I need to ignore whatever iter((list) this sampler generates and instead control the loaded samples in my dataset class with a few class members. That makes the idx parameter passed to getitem useless. This code was modified from mmdetection.
  def get_curr_idx(self):

        if self.idx == 0:
            random.shuffle(self.order)

        curr_idx = self.order[self.idx]
        self.idx = (self.idx + 1) % self.total_samples

        return curr_idx

    def __getitem__(self, idx):

        curr_idx = self.get_curr_idx()

        if self.test_mode:
            return self.prepare_test_img(curr_idx)
        while True:
            data = self.prepare_train_img(curr_idx)
            if data is None:
                curr_idx = self.get_curr_idx()
            else:
                return data