Hi there,
I have been running a script to train a pretrained transformer on a summarization task. I am using custom data which I have put into my S3 bucket which is also the default bucket for this job.
I have been getting this error in return, and have not been able to figure out what the solution is. I have run the exact same script on the xsum dataset just to see if it’s the custom dataset that is the issue, and I can indeed confirm that the job works when using the xsum dataset.
from sagemaker.huggingface import HuggingFace

hyperparameters={
    'model_name_or_path': 'google/pegasus-large',
    'train_file': "/opt/ml/input/data/train/final_aws_deepgram_train.csv",
    'test_file': "/opt/ml/input/data/test/final_aws_deepgram_test.csv",
    'validation_file': "/opt/ml/input/data/validation/final_aws_deepgram_validation.csv",
    'text_column': 'document',
    'summary_column': 'summary',
    'do_train': True,
    'do_eval': True,
    'fp16': True,
    'per_device_train_batch_size': 2,
    'per_device_eval_batch_size': 2,
    'evaluation_strategy': "steps",
    'eval_steps': 200,
    'weight_decay': 0.01,
    'learning_rate': 2e-5,
    'max_grad_norm': 1,
    'max_steps': 200,
    'max_source_length': 500,
    'max_target_length': 100,
    'load_best_model_at_end': True,
    'output_dir': '/opt/ml/model'
}

# git configuration to download our fine-tuning script
git_config = {'repo': 'https://github.com/huggingface/transformers.git', 'branch': 'v4.6.1'} #'branch': 'v4.6.1'

# instance configurations
instance_type='ml.p3.2xlarge'
instance_count=1
volume_size=200

# metric definition to extract the results
metric_definitions=[
     {"Name": "train_runtime", "Regex": "train_runtime.*=\D*(.*?)$"},
     {'Name': 'train_samples_per_second', 'Regex': "train_samples_per_second.*=\D*(.*?)$"}
]

huggingface_estimator = HuggingFace(entry_point='run_summarization_original.py',
                                    source_dir='transformers/examples/pytorch/summarization',
                                    git_config=git_config,
                                    metric_definitions=metric_definitions,
                                    instance_type=instance_type,
                                    instance_count=instance_count,
                                    volume_size=volume_size,
                                    role=role,
                                    transformers_version='4.6.1',
                                    pytorch_version='1.7.1',
                                    py_version='py36',
                                    hyperparameters = hyperparameters)

# starting the train job
huggingface_estimator.fit(
  {'train': 's3://qfn-transcription/ujjawal_files/final_aws_deepgram_train.csv',
   'test': 's3://qfn-transcription/ujjawal_files/final_aws_deepgram_test.csv',
  'validation': 's3://qfn-transcription/ujjawal_files/final_aws_deepgram_validate.csv'}
)

2021-06-22 15:34:39 Starting - Starting the training job...
2021-06-22 15:35:03 Starting - Launching requested ML instancesProfilerReport-1624376073: InProgress
.........
2021-06-22 15:36:33 Starting - Preparing the instances for training.........
2021-06-22 15:38:06 Downloading - Downloading input data
2021-06-22 15:38:06 Training - Downloading the training image.....................
2021-06-22 15:41:34 Uploading - Uploading generated training model
2021-06-22 15:41:34 Failed - Training job failed
..
---------------------------------------------------------------------------
UnexpectedStatusException                 Traceback (most recent call last)
<ipython-input-7-ca8819244de5> in <module>
      3   {'train': 's3://qfn-transcription/ujjawal_files/final_aws_deepgram_train.csv',
      4    'test': 's3://qfn-transcription/ujjawal_files/final_aws_deepgram_test.csv',
----> 5   'validation': 's3://qfn-transcription/ujjawal_files/final_aws_deepgram_validate.csv'}
      6 )

~/anaconda3/envs/python3/lib/python3.6/site-packages/sagemaker/estimator.py in fit(self, inputs, wait, logs, job_name, experiment_config)
    680         self.jobs.append(self.latest_training_job)
    681         if wait:
--> 682             self.latest_training_job.wait(logs=logs)
    683 
    684     def _compilation_job_name(self):

~/anaconda3/envs/python3/lib/python3.6/site-packages/sagemaker/estimator.py in wait(self, logs)
   1623         # If logs are requested, call logs_for_jobs.
   1624         if logs != "None":
-> 1625             self.sagemaker_session.logs_for_job(self.job_name, wait=True, log_type=logs)
   1626         else:
   1627             self.sagemaker_session.wait_for_job(self.job_name)

~/anaconda3/envs/python3/lib/python3.6/site-packages/sagemaker/session.py in logs_for_job(self, job_name, wait, poll, log_type)
   3683 
   3684         if wait:
-> 3685             self._check_job_status(job_name, description, "TrainingJobStatus")
   3686             if dot:
   3687                 print()

~/anaconda3/envs/python3/lib/python3.6/site-packages/sagemaker/session.py in _check_job_status(self, job, desc, status_key_name)
   3243                 ),
   3244                 allowed_statuses=["Completed", "Stopped"],
-> 3245                 actual_status=status,
   3246             )
   3247 

UnexpectedStatusException: Error for Training job huggingface-pytorch-training-2021-06-22-15-34-33-634: Failed. Reason: AlgorithmError: ExecuteUserScriptError:
Command "/opt/conda/bin/python3.6 run_summarization_original.py --do_eval True --do_train True --eval_steps 200 --evaluation_strategy steps --fp16 True --learning_rate 2e-05 --load_best_model_at_end True --max_grad_norm 1 --max_source_length 500 --max_steps 200 --max_target_length 100 --model_name_or_path google/pegasus-large --output_dir /opt/ml/model --per_device_eval_batch_size 2 --per_device_train_batch_size 2 --summary_column summary --test_file /opt/ml/input/data/test/final_aws_deepgram_test.csv --text_column document --train_file /opt/ml/input/data/train/final_aws_deepgram_train.csv --validation_file /opt/ml/input/data/validation/final_aws_deepgram_validation.csv --weight_decay 0.01"
Traceback (most recent call last):
  File "run_summarization_original.py", line 606, in <module>
    main()
  File "run_summarization_original.py", line 325, in main
    datasets = load_dataset(extension, data_files=data_files, cache_dir=model_args.cache_dir)
  File "/opt/conda/lib/p

Thanks!