Here’s an update to the model code:
from transformers import GPT2LMHeadModel
from FeatureExtraction.NGrams import *
import time
from transformers import GPT2Tokenizer


class GPT2FinetunedWithNgrams(GPT2LMHeadModel):
    def __init__(self, config):
        super().__init__(config)
        self.tokenizer = GPT2Tokenizer.from_pretrained('gpt2')

    def load_ngrams_model(self, ngrams_model_path):
        self.ngrams_model = NGrams(ngrams_model_path)

    def forward(
            self,
            input_ids=None,
            past=None,
            attention_mask=None,
            token_type_ids=None,
            position_ids=None,
            head_mask=None,
            inputs_embeds=None,
            labels=None,
            use_cache=True,
    ):
  
      transformer_outputs = self.transformer(
            input_ids,
            past=past,
            attention_mask=attention_mask,
            token_type_ids=token_type_ids,
            position_ids=position_ids,
            head_mask=head_mask,
            inputs_embeds=inputs_embeds,
            use_cache=use_cache,
        )

        hidden_states = transformer_outputs[0]
        lm_logits = self.lm_head(hidden_states)

        # Somehow convert lm_logits to new_input_ids. Then can do the following:
        # gpt2_sent_ids = super().generate(new_input_ids, max_length=50) -> generates a tensor of token_ids for the generated sentence by gpt2.
        # gpt2_sent_str = self.tokenizer.decode(gpt2_sent_ids[0], skip_special_tokens=True) -> will take the tensor of token_ids and convert it to the string.
        # loss = self.ngrams_model.sentence_loss(gpt2_sent) -> the ngrams model uses a string as input to calculate a loss.
        # return (loss, lm_logits)

I think that will work for what I need, but I welcome feedback from others. I’ve been looking through the documentation and can’t seem to find a way to convert lm_logits into new_input_ids. That’ll take care of the first question. I still don’t understand how to pass my dataset to Trainer.