Thanks for the answer!
I haven’t seen anything strange in the training loop:
    with tqdm(total=len(train_generator)) as prbar:
        for batch_idx, batch in enumerate(train_generator, 1):
            output = self._model(batch[:3]).squeeze(dim=1)
            target = batch[3].to(self._device)
            loss = loss_func(output, target)
            loss.backward()
            optimizer.step()
            optimizer.zero_grad()

            total_loss += float(loss.item()) * target.size(0)
            y_true.extend(list(target.data.cpu().numpy()))
            y_pred.extend(list(output.data.cpu().numpy()))
            l1_part += float(l1_weight * weights_norm_1 / float(loss) * 100)
            l2_part += float(l2_weight * weights_norm_2 / float(loss) * 100)

Can any things be in the impelemnted model class that can slows the perofmance?