I went through the documentation/course/videos in more detail and ended up making some updates to the train.py. I believe possibly the memory error was due to using max padding lengh and the default datacollator instead of one with padding. I fixed that and removed all the the train args so it is not trying to run evaluation. It looks like I’ve come full cirlce now, as I’m getting the same ‘loss’ key error in sagemaker as I’m getting locally.
See below for the full training script and notebook execution code, but the error is the same as what I first mentioned in the beginning of this thread:

image1884×830 198 KB

Any idea how I can get around this? I did the full pyrtorch implementation from here and got

AttributeError: ‘Seq2SeqModelOutput’ object has no attribute ‘loss’

So it appears it’s defaulting to the wrong type? I can share the code for that as well, but it’s essentially the same but I’m just using ‘facebook/bart-large-mnli’ and unlabelled data. I’m at a “loss” of what to try next. ha. ha.
train.py
from transformers import (
    AutoModel,
    Trainer,
    TrainingArguments,
    AutoTokenizer,
    AutoFeatureExtractor,
    default_data_collator,
)

from sklearn.metrics import accuracy_score, precision_recall_fscore_support
from datasets import load_dataset
import random
import logging
import sys
import argparse
import os
import torch

if __name__ == "__main__":

    parser = argparse.ArgumentParser()

    # hyperparameters sent by the client are passed as command-line arguments to the script.
    parser.add_argument("--epochs", type=int, default=3)
    parser.add_argument("--train_batch_size", type=int, default=32)
    parser.add_argument("--eval_batch_size", type=int, default=64)
    parser.add_argument("--warmup_steps", type=int, default=500)
    parser.add_argument("--model_id", type=str)
    parser.add_argument("--learning_rate", type=str, default=5e-5)
    parser.add_argument("--train_file", type=str, default="train_text.json")
    parser.add_argument("--test_file", type=str, default="test_text.json")
    parser.add_argument("--fp16", type=bool, default=True)

    # Data, model, and output directories
    parser.add_argument("--output_data_dir", type=str, default=os.environ["SM_OUTPUT_DATA_DIR"])
    parser.add_argument("--model_dir", type=str, default=os.environ["SM_MODEL_DIR"])
    parser.add_argument("--n_gpus", type=str, default=os.environ["SM_NUM_GPUS"])
    parser.add_argument("--training_dir", type=str, default=os.environ["SM_CHANNEL_TRAIN"])
    parser.add_argument("--test_dir", type=str, default=os.environ["SM_CHANNEL_TEST"])

    args, _ = parser.parse_known_args()

    # Set up logging
    logger = logging.getLogger(__name__)

    logging.basicConfig(
        level=logging.getLevelName("INFO"),
        handlers=[logging.StreamHandler(sys.stdout)],
        format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
    )
    
    print('\nWalk:')
    for path, subdirs, files in os.walk('/opt/ml'): 
        for name in files: print(os.path.join(path, name))
          

    # load datasets
    raw_train_dataset = load_dataset("json", data_files=os.path.join(args.training_dir, args.train_file))["train"]
    raw_test_dataset = load_dataset("json", data_files=os.path.join(args.test_dir, args.test_file))["train"]

    # load tokenizer
    tokenizer = AutoTokenizer.from_pretrained(args.model_id)

    # preprocess function, tokenizes text
    def preprocess_function(examples):
        return tokenizer(examples["inputs"], truncation=True)

    # preprocess dataset
    train_dataset = raw_train_dataset.map(
        preprocess_function,
        batched=True,
    )
    test_dataset = raw_test_dataset.map(
        preprocess_function,
        batched=True,
    )


    # print size
    logger.info(f" loaded train_dataset length is: {len(train_dataset)}")
    logger.info(f" loaded test_dataset length is: {len(test_dataset)}")

    

    print('\nargs.model_id', args.model_id)
    # download model from model hub
    model = AutoModel.from_pretrained(args.model_id)

    # define training args
    training_args = TrainingArguments(
        output_dir=args.model_dir,
#         num_train_epochs=args.epochs,
#         per_device_train_batch_size=args.train_batch_size,
#         per_device_eval_batch_size=args.eval_batch_size,
#         warmup_steps=args.warmup_steps,
#         fp16=args.fp16,
#         evaluation_strategy="epoch",
#         save_strategy="epoch",
#         logging_dir=f"{args.output_data_dir}/logs",
#         learning_rate=float(args.learning_rate),
#         load_best_model_at_end=True,
#         metric_for_best_model="f1",
    )

    # create Trainer instance
    trainer = Trainer(
        model=model,
        args=training_args,
        train_dataset=train_dataset,
        eval_dataset=test_dataset,
        tokenizer=tokenizer,
#         data_collator=default_data_collator,
    )

    # train model
    trainer.train()
    

    # evaluate model
    eval_result = trainer.evaluate(eval_dataset=test_dataset)

    # writes eval result to file which can be accessed later in s3 ouput
    with open(os.path.join(args.output_data_dir, "eval_results.txt"), "w") as writer:
        print(f"***** Eval results *****")
        for key, value in sorted(eval_result.items()):
            writer.write(f"{key} = {value}\n")


    # Saves the model to s3
    trainer.save_model(args.model_dir)


.
.
.
.
notebook execution:
!pip install "sagemaker>=2.48.0"


import sagemaker

sess = sagemaker.Session()
# sagemaker session bucket -> used for uploading data, models and logs
# sagemaker will automatically create this bucket if it not exists
sagemaker_session_bucket=None
if sagemaker_session_bucket is None and sess is not None:
    # set to default bucket if a bucket name is not given
    sagemaker_session_bucket = sess.default_bucket()

role = sagemaker.get_execution_role()
sess = sagemaker.Session(default_bucket=sagemaker_session_bucket)

print(f"sagemaker role arn: {role}")
print(f"sagemaker bucket: {sess.default_bucket()}")
print(f"sagemaker session region: {sess.boto_region_name}")

# Download some training data

!wget https://github.com/saurabh3949/Text-Classification-Datasets/raw/master/dbpedia_csv.tar.gz
!tar -xzvf dbpedia_csv.tar.gz

import pandas as pd
import json
import os

# Write small train and test files

df = pd.read_csv('dbpedia_csv/train.csv', header = None)

# write as small train input file
with open('train_text.json', 'w') as outfile:
    for desc in df.iloc[:10000, 2]:
        json.dump({"inputs": desc}, outfile)
        outfile.write('\n')
        
with open('test_text.json', 'w') as outfile:
    for desc in df.iloc[10000:15000, 2]:
        json.dump({"inputs": desc}, outfile)
        outfile.write('\n')

from sagemaker.s3 import S3Uploader

s3_prefix = 'batch-data'

training_input_path = f's3://{sess.default_bucket()}/{s3_prefix}/train'
test_input_path = f's3://{sess.default_bucket()}/{s3_prefix}/test'


# upload datasets
train_remote = S3Uploader.upload('train_text.json',training_input_path)
test_remote = S3Uploader.upload('test_text.json',test_input_path)

print(f"train dataset uploaded to: \n{train_remote}\n{test_remote}")


from sagemaker.huggingface import HuggingFace
import time

mid = 'facebook/bart-large-mnli'




# hyperparameters, which are passed into the training job
hyperparameters={'epochs': 1,                          # number of training epochs
                 'train_batch_size': 32,               # batch size for training
                 'eval_batch_size': 64,                # batch size for evaluation
                 'learning_rate': 3e-5,                # learning rate used during training
                 'model_id':mid, # pre-trained model
                 'fp16': True,                         # Whether to use 16-bit (mixed) precision training
                 'train_file': 'train_text.json',    # training dataset
                 'test_file': 'test_text.json',      # test dataset
                 }


metric_definitions=[
    {'Name': 'eval_loss',               'Regex': "'eval_loss': ([0-9]+(.|e\-)[0-9]+),?"},
    {'Name': 'eval_accuracy',           'Regex': "'eval_accuracy': ([0-9]+(.|e\-)[0-9]+),?"},
    {'Name': 'eval_f1',                 'Regex': "'eval_f1': ([0-9]+(.|e\-)[0-9]+),?"},
    {'Name': 'eval_precision',          'Regex': "'eval_precision': ([0-9]+(.|e\-)[0-9]+),?"}]


# define Training Job Name 
job_name = f'hf--{mid.replace("/", "-")}--{time.strftime("%H-%M-%S", time.localtime())}'

instance = 'ml.p3.2xlarge'


# create the Estimator
huggingface_estimator = HuggingFace(
    entry_point          = 'train.py',        # fine-tuning script used in training jon
    source_dir           = 'scripts',      # directory where fine-tuning script is stored
    instance_type        = instance,   # instances type used for the training job
    instance_count       = 1,                 # the number of instances used for training
    base_job_name        = job_name,          # the name of the training job
    role                 = role,              # Iam role used in training job to access AWS ressources, e.g. S3
    transformers_version = '4.6',             # the transformers version used in the training job
    pytorch_version      = '1.7',             # the pytorch_version version used in the training job
    py_version           = 'py36',            # the python version used in the training job
    hyperparameters      = hyperparameters,   # the hyperparameter used for running the training job
    metric_definitions   = metric_definitions # the metrics regex definitions to extract logs
)

# define a data input dictonary with our uploaded s3 uris
training_data = {
    'train': train_remote,
    'test': test_remote
}

# starting the train job with our uploaded datasets as input
huggingface_estimator.fit(training_data, wait=True)