Command:
export d=mbart_benchmark_data
python examples/benchmarking/run_benchmark.py \
  --models facebook/bart-large-cnn \
  --log_filename $d/log.txt \
  --inference_memory_csv \
  $d/inference_memory.csv \
  --train_memory_csv $d/train_memory.csv \
  --train_time_csv $d/train_time.csv \
  --inference_time_csv $d/inference_time.csv \
  --fp16 --log_print --training  --save_to_csv \
  --batch_sizes 4 8 12 16

Results:
bart-large-cnn
==============        TRAIN - MEMORY - RESULTS        =======
--------------------------------------------------------------------------------
 Model Name                     Batch Size     Seq Length    Mem in MB
--------------------------------------------------------------------------------
   facebook/bart-large-cnn           4              8             2795
   facebook/bart-large-cnn           4              32           2897
   facebook/bart-large-cnn           4             128           3169
   facebook/bart-large-cnn           4             512           6873
   facebook/bart-large-cnn           8              8             2827
   facebook/bart-large-cnn           8              32            2933
   facebook/bart-large-cnn           8             128            3465
   facebook/bart-large-cnn           8             512           12195          
   facebook/bart-large-cnn           12             8             2859
   facebook/bart-large-cnn           12             32            3137
   facebook/bart-large-cnn           12            128            4371
   facebook/bart-large-cnn           12            512            N/A
   facebook/bart-large-cnn           16             8             2891
   facebook/bart-large-cnn           16             32            3105
   facebook/bart-large-cnn           16            128            5153
   facebook/bart-large-cnn           16            512            N/A
--------------------------------------------------------------------------------

mbart
=========       TRAIN - MEMORY - RESULTS        =======
--------------------------------------------------------------------------------
          Model Name             Batch Size     Seq Length    Memory in MB
--------------------------------------------------------------------------------
  facebook/mbart-large-en-ro         4              8             4355
  facebook/mbart-large-en-ro         4              32            4947
  facebook/mbart-large-en-ro         4             128            5117
  facebook/mbart-large-en-ro         4             512           10383
  facebook/mbart-large-en-ro         8              8             4877
  facebook/mbart-large-en-ro         8              32            4493
  facebook/mbart-large-en-ro         8             128            5857
  facebook/mbart-large-en-ro         8             512            N/A
  facebook/mbart-large-en-ro         12             8             4909
  facebook/mbart-large-en-ro         12             32            5085
  facebook/mbart-large-en-ro         12            128            7079
  facebook/mbart-large-en-ro         12            512            N/A
  facebook/mbart-large-en-ro         16             8             4941
  facebook/mbart-large-en-ro         16             32            4663
  facebook/mbart-large-en-ro         16            128            8655
  facebook/mbart-large-en-ro         16            512            N/A
--------------------------------------------------------------------------------