I noticed a strange memory allocation on the GPU

tesla_t4_4722×604 61 KB

training_args = TrainingArguments(
    output_dir="./rus_model",
    group_by_length=True,
    per_device_train_batch_size=16,
    gradient_accumulation_steps=2,
    evaluation_strategy="steps",
    num_train_epochs=30,
    fp16=True,
    save_steps=400,
    eval_steps=200,
    logging_steps=100,
    learning_rate=3e-4,
    warmup_steps=500,
    save_total_limit=2,
    dataloader_num_workers=16,
    report_to='tensorboard'
)