RoversX/midjourney-falcon-7b-beta1

Model Card for Model ID

Using Google Colab

training_args = transformers.TrainingArguments(
      per_device_train_batch_size=1,
      gradient_accumulation_steps=4,
      num_train_epochs=3,
      learning_rate=2e-4,
      fp16=True,
      save_total_limit=3,
      logging_steps=1,
      output_dir="experiments",
      optim="paged_adamw_8bit",
      lr_scheduler_type="cosine",
      warmup_ratio=0.05,
)

trainer = transformers.Trainer(
    model=model,
    train_dataset=data,
    args=training_args,
    data_collator=transformers.DataCollatorForLanguageModeling(tokenizer, mlm=False)
)
model.config.use_cache = False
trainer.train()

TrainOutput(global_step=216, training_loss=1.8804574598454766, metrics={'train_runtime': 306.7499, 'train_samples_per_second': 2.826, 'train_steps_per_second': 0.704, 'total_flos': 1277315019871488.0, 'train_loss': 1.8804574598454766, 'epoch': 2.99})