Batch generation is now possible for GPT2 in master by leveraging the functionality shown in this PR: https://github.com/huggingface/transformers/pull/7552?notification_referrer_id=MDE4Ok5vdGlmaWNhdGlvblRocmVhZDEyMTMzNzA0MDA6MjM0MjM2MTk%3D#event-3876130796 156 .
For more info on how to prepare a GPT2 for batch generation, you can checkout this test:

  
      github.com
  
  
    huggingface/transformers/blob/890e790e16084e58a1ecb9329c98ec3e76c45994/tests/test_modeling_gpt2.py#L430 95


def test_gpt2_sequence_classification_model(self):
    config_and_inputs = self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_gpt2_for_sequence_classification(*config_and_inputs)

def test_gpt2_gradient_checkpointing(self):
    config_and_inputs = self.model_tester.prepare_config_and_inputs(gradient_checkpointing=True)
    self.model_tester.create_and_check_forward_and_backwards(*config_and_inputs)

@slow
def test_batch_generation(self):
    model = GPT2LMHeadModel.from_pretrained("gpt2")
    model.to(torch_device)
    tokenizer = GPT2Tokenizer.from_pretrained("gpt2")

    tokenizer.padding_side = "left"

    # Define PAD Token = EOS Token = 50256
    tokenizer.pad_token = tokenizer.eos_token
    model.config.pad_token_id = model.config.eos_token_id