Thanks! Indeed this is the case. The last values of preds, which is passed to the compute_metrics function, were all equal to -100, the padding index.
Here is a reduced version of my setup that produces arrays with padded values only:


Code
import torch
import torch.nn as nn
from torch.utils.data import Dataset
from transformers import TrainingArguments, Trainer
import numpy as np
from sklearn.metrics import accuracy_score, f1_score
class CustomDataset(Dataset):
def __init__(self):
    
    self.input = [[0.,0.,0.,1.],[1.,0.,0.,1.],[1.,1.,1.,0.],[0.,0.,0.,1.],
                  [1.,0.,0.,1.],[1.,1.,1.,0.],[1.,0.,0.,1.],[1.,1.,1.,0.]]
    self.labels = [1,1,0,1,1,0,1,1]
    self.n_tokens = 4
    self.n_labels = 2
    
def __len__(self):
    
    return(len(self.input))

def __getitem__(self, idx):
    
    input_dict = {"inputs": torch.tensor(self.input[idx]),
        "label_ids": torch.tensor(self.labels[idx])}
    return input_dict

class MyTrainer(Trainer):
def compute_loss(self, model, inputs, return_outputs = False):
labels = inputs[“labels”].long()
logits = model(inputs[“inputs”])
loss_function = nn.CrossEntropyLoss()
loss = loss_function(logits,labels)
return (loss, logits) if return_outputs else loss
def compute_metrics(eval_pred):
preds, labels = eval_pred
print(preds)
preds = np.argmax(preds, axis=1)
accuracy = accuracy_score(labels, preds)
micro_f1 = f1_score(labels, preds, average = "micro")
macro_f1 = f1_score(labels, preds, average = "macro")

return {"Accuracy": accuracy, "Micro F1": micro_f1, "Macro F1": macro_f1}

dataset = CustomDataset()
n_tokens = dataset.n_tokens
n_hidden = 2
n_labels = dataset.n_labels
model = nn.Sequential(
nn.Linear(n_tokens,n_hidden),
nn.ReLU(),
nn.BatchNorm1d(n_hidden),
nn.Linear(n_hidden,n_hidden),
nn.ReLU(),
nn.BatchNorm1d(n_hidden),
nn.Linear(n_hidden, n_labels))
train_dataset = eval_dataset = dataset
args = TrainingArguments(output_dir = “example”,
report_to=[],
num_train_epochs = 3,
per_device_train_batch_size =2,
per_device_eval_batch_size = 1,
evaluation_strategy = “steps”,
logging_steps =4)
trainer = MyTrainer(
model = model,
args=args,
train_dataset=train_dataset,
eval_dataset=eval_dataset,
compute_metrics=compute_metrics
)
trainer.train()

Also I found that the shape of preds changes for different values of per_device_eval_batch_size . If I set eval_batch_size=2 in this setup, my compute_metrics function fails at argmax(axis =1), because the array is one-dimensional.
Generally, preds,


as it is used here,
preds = preds_gatherer.finalize() if not prediction_loss_only else None
metrics = self.compute_metrics(EvalPrediction(predictions=preds, label_ids=label_ids))

should simply be the (concatenated) model output, did I understand this correctly?