I’m pre training a distillBert model from scratch and saving the model every 300 steps , When trying to load a checkpoint to continue training from the Trainer show that it’s skipping the trained steps but it just starts from 0 and doesn’t start logging or saving until the trainer passes the number of skipped steps.
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model = DistilBertForMaskedLM.from_pretrained("/content/drive/My Drive/AIMBert/output_gpu/checkpoint-48000",config=config).to(device)

training_args = TrainingArguments(
output_dir="/content/drive/My Drive/AIMBert/output_gpu",
logging_dir='/content/drive/My Drive/AIMBert/logs_gpu',
overwrite_output_dir=True,
num_train_epochs=10,
per_device_train_batch_size=32,
per_device_eval_batch_size = 32,
logging_steps = 100,
save_steps=300,
save_total_limit=5,
evaluation_strategy = "steps",
eval_steps=50000,
seed = 42,
prediction_loss_only=True,
fp16 = True
)

trainer = Trainer(
model=model,
args=training_args,
data_collator=data_collator,
train_dataset= processed_train_dataset,
eval_dataset = processed_valid_dataset
)

trainer.train('/content/drive/My Drive/AIMBert/output_gpu/checkpoint-48000')