GPT2  model training , Loss nan
### System Info

Sometimes I get this error.  while following this article about fine-tuning based on question and answers. [https://discuss.huggingface.co/t/fine-tuning-gpt2-for-question-answering/31895](url)

I have just updated the code to create batches because my dataset is more extensive. 
Here is the code that I am using:







### Who can help?

_No response_

### Information

- [X] The official example scripts
- [X] My own modified scripts

### Tasks

- [ ] An officially supported task in the `examples` folder (such as GLUE/SQuAD, ...)
- [ ] My own task or dataset (give details below)

### Reproduction

import pandas as pd
import torch
from torch.utils.data import DataLoader, Dataset
from transformers import GPT2Tokenizer, GPT2LMHeadModel

class FeedbackEssentials(Dataset):
    def __init__(self, qa_pairs, tokenizer, max_length):
        self.qa_pairs = qa_pairs
        self.tokenizer = tokenizer
        self.max_length = max_length

    def __len__(self):
        return len(self.qa_pairs)

    def __getitem__(self, idx):
        question = self.qa_pairs[idx][0]
        text = f"{question} {self.tokenizer.eos_token}"
        input_ids = self.tokenizer.encode(text, add_special_tokens=True, max_length=self.max_length, padding='max_length', truncation=True)
        attention_mask = [1] * len(input_ids)  # Assuming all tokens should be attended to

        return {
            'input_ids': torch.tensor(input_ids),
            'attention_mask': torch.tensor(attention_mask)
        }

train_df = pd.read_csv('/Users/irfanyaqub/Downloads/Research Dataset/train_dataset.csv')
val_df = pd.read_csv('/Users/irfanyaqub/Downloads/Research Dataset/val_dataset.csv')
val_df=val_df[:10]


def remove_anomalies(value):
    return value['Coding'].replace({'\*-': ''}, regex=True)

train_df['Coding'] = remove_anomalies(train_df)
val_df['Coding'] = remove_anomalies(val_df)

tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
model = GPT2LMHeadModel.from_pretrained('gpt2')
tokenizer.add_special_tokens({'pad_token': '[PAD]'})
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

def text_manipulation(train_dataset):
    column1_values = train_dataset['Total Marks'].values
    column2_values = train_dataset['Coding'].values
    listOfLists = [[pair[0], pair[1]] for pair in zip(column1_values, column2_values)]

    text = ""
    for feedback in listOfLists:
        text += f"{feedback[0]} {feedback[1]} {tokenizer.eos_token}"
    return text

training_dataset = text_manipulation(val_df)
max_length_training = max(len(tokenizer.encode(qa_pair[0], add_special_tokens=True)) for qa_pair in training_dataset)
dataset_training = FeedbackEssentials(training_dataset, tokenizer, max_length_training)

batch_size = 4
dataloader = DataLoader(dataset_training, batch_size=batch_size, shuffle=True)

optimizer = torch.optim.AdamW(model.parameters(), lr=5e-5)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=0.9)

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model.to(device)
model.train()

for epoch in range(1):
    for batch in dataloader:
        input_ids = batch['input_ids'].to(device)
        attention_mask = batch['attention_mask'].to(device)

        optimizer.zero_grad()
        loss = model(input_ids.to(device), labels=input_ids.to(device))[0]
        loss.backward()
        optimizer.step()
        scheduler.step()

    if epoch % 100 == 0:
        print(f"Epoch {epoch}, Loss {loss.item()}")

model.eval()

def generate_response(question):
    input_ids = tokenizer.encode(question, add_special_tokens=True, return_tensors='pt').to(device)
    sample_output = model.generate(input_ids, do_sample=True, max_length=200, top_k=20, top_p=1.0)

    answer = tokenizer.decode(sample_output[0], skip_special_tokens=True)
    sentences = answer.split('. ')

    for sentence in sentences:
        if question in sentence:
            return sentence

    return answer




### Expected behavior

Expected output will be something like that:
question = “How to delete an account”
response = generate_response(question)
print(f"{question}\n {response}")

Answer: How to delete an account
How to delete an account <|question|>

This tool allows you to:
Hi @irfan767, thanks for raising an issue! 

Questions about debugging code or custom training behaviour are best placed in our [forums](https://discuss.huggingface.co/). We try to reserve the github issues for feature requests and bug reports.