I have a simple but fairly awful approach I’m using. My issue is that I can’t figure out how to predict the size of the model in advance, so I can’t automatically determine the appropriate batch size. Any thoughts?
# taken from somewhere online
def chunk_list(lst, n):
    """Yield successive n-sized chunks from lst."""
    for i in range(0, len(lst), n):
        yield lst[i:i + n]


class ZeroShotPipelineMiniBatch():

    def __init__(self, zs_pipeline, max_input_size=40):
        self.MAX_INPUT_SIZE = max_input_size
        self.pipeline = zs_pipeline

    def __call__(self, inputs, label_list, *args, **kwargs):
        batch_size = int(np.floor(self.MAX_INPUT_SIZE / len(label_list)))

        ret = []
        for chunk in chunk_list(inputs, batch_size):
            r = self.pipeline(chunk, label_list, *args, **kwargs)
            if isinstance(r, list):
                ret += r
            else:
                ret += [r]
        return ret