I think I found a solution!
class MySampler(Sampler):
    """
    Sampler that does not screw around with taking too early/late indices.
    """
    def __init__(self, data_source, input_config):
        self.config = input_config
        self.data_source = data_source
        super().__init__(data_source)

    def __iter__(self):
        return iter([i for i in range(len(self.data_source)) if i >= self.config.num_conseq_frames - 1 and
                    len(self.data_source) - i > self.config.num_conseq_frames])

    def __len__(self):
        len(self.data_source)