Could try this with the examples/seq2seq scripts ?
also we have recently added Trainer support for seq2seq tasks as well.
see

  
      github.com
  
  
    huggingface/transformers/blob/master/examples/seq2seq/finetune_trainer.py 224
import logging
import os
import sys
from dataclasses import dataclass, field
from typing import Optional

from seq2seq_trainer import Seq2SeqTrainer, arg_to_scheduler_choices
from transformers import (
    AutoConfig,
    AutoModelForSeq2SeqLM,
    AutoTokenizer,
    HfArgumentParser,
    MBartTokenizer,
    TrainingArguments,
    set_seed,
)
from transformers.trainer_utils import EvaluationStrategy
from utils import (
    LegacySeq2SeqDataset,
    Seq2SeqDataCollator,


  This file has been truncated. show original

  
  
    
    
  
  


and

  
      
      GitHub
  
  
    

huggingface/transformers 39

🤗Transformers: State-of-the-art Natural Language Processing for Pytorch and TensorFlow 2.0. - huggingface/transformers