Model save

README.md

@@ -0,0 +1,73 @@
+---
+library_name: transformers
+tags:
+- trl
+- dpo
+- generated_from_trainer
+model-index:
+- name: zephyr-7b-gemma-dpo
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# zephyr-7b-gemma-dpo
+
+This model was trained from scratch on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.4794
+- Rewards/chosen: 0.1794
+- Rewards/rejected: -0.8259
+- Rewards/accuracies: 0.7396
+- Rewards/margins: 1.0054
+- Logps/rejected: -348.6507
+- Logps/chosen: -332.3037
+- Logits/rejected: -3.0577
+- Logits/chosen: -3.0380
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 4
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 128
+- total_eval_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 2
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.4901        | 0.9858 | 52   | 0.5042          | 0.4415         | -0.3953          | 0.7708             | 0.8368          | -340.0374      | -327.0610    | -3.1187         | -3.1021       |
+| 0.1326        | 1.9716 | 104  | 0.4794          | 0.1794         | -0.8259          | 0.7396             | 1.0054          | -348.6507      | -332.3037    | -3.0577         | -3.0380       |
+
+
+### Framework versions
+
+- Transformers 4.45.2
+- Pytorch 2.5.1+rocm6.2
+- Datasets 3.5.0
+- Tokenizers 0.20.3

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.971563981042654,
+    "total_flos": 0.0,
+    "train_loss": 0.3703585032087106,
+    "train_runtime": 3327.4148,
+    "train_samples": 6750,
+    "train_samples_per_second": 4.057,
+    "train_steps_per_second": 0.031
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.45.2"
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.971563981042654,
+    "total_flos": 0.0,
+    "train_loss": 0.3703585032087106,
+    "train_runtime": 3327.4148,
+    "train_samples": 6750,
+    "train_samples_per_second": 4.057,
+    "train_steps_per_second": 0.031
+}

trainer_state.json

@@ -0,0 +1,239 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.971563981042654,
+  "eval_steps": 500,
+  "global_step": 104,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.018957345971563982,
+      "grad_norm": 84.63106282662845,
+      "learning_rate": 4.545454545454545e-08,
+      "logits/chosen": -2.8878073692321777,
+      "logits/rejected": -2.898465633392334,
+      "logps/chosen": -425.2048034667969,
+      "logps/rejected": -484.02008056640625,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.1895734597156398,
+      "grad_norm": 78.74620395904364,
+      "learning_rate": 4.545454545454545e-07,
+      "logits/chosen": -2.871765375137329,
+      "logits/rejected": -2.8902745246887207,
+      "logps/chosen": -346.8144836425781,
+      "logps/rejected": -398.9815368652344,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.007920680567622185,
+      "rewards/margins": 0.008156180381774902,
+      "rewards/rejected": -0.0002354999742237851,
+      "step": 10
+    },
+    {
+      "epoch": 0.3791469194312796,
+      "grad_norm": 69.92815473359221,
+      "learning_rate": 4.885348141000122e-07,
+      "logits/chosen": -2.9278335571289062,
+      "logits/rejected": -2.9445371627807617,
+      "logps/chosen": -390.162841796875,
+      "logps/rejected": -417.49310302734375,
+      "loss": 0.6416,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": 0.1379609853029251,
+      "rewards/margins": 0.1072385162115097,
+      "rewards/rejected": 0.030722463503479958,
+      "step": 20
+    },
+    {
+      "epoch": 0.5687203791469194,
+      "grad_norm": 68.85043622104293,
+      "learning_rate": 4.5025027361734613e-07,
+      "logits/chosen": -3.0001819133758545,
+      "logits/rejected": -2.987617015838623,
+      "logps/chosen": -334.628662109375,
+      "logps/rejected": -352.9248962402344,
+      "loss": 0.5519,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": 0.35683774948120117,
+      "rewards/margins": 0.3989708423614502,
+      "rewards/rejected": -0.04213308170437813,
+      "step": 30
+    },
+    {
+      "epoch": 0.7582938388625592,
+      "grad_norm": 62.898371390545925,
+      "learning_rate": 3.893311157806091e-07,
+      "logits/chosen": -3.067532539367676,
+      "logits/rejected": -3.0627801418304443,
+      "logps/chosen": -364.905517578125,
+      "logps/rejected": -350.44317626953125,
+      "loss": 0.5539,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.5169381499290466,
+      "rewards/margins": 0.7471343278884888,
+      "rewards/rejected": -0.23019620776176453,
+      "step": 40
+    },
+    {
+      "epoch": 0.9478672985781991,
+      "grad_norm": 64.78933961282183,
+      "learning_rate": 3.126631330646801e-07,
+      "logits/chosen": -3.0639877319335938,
+      "logits/rejected": -3.0641586780548096,
+      "logps/chosen": -368.72076416015625,
+      "logps/rejected": -416.376953125,
+      "loss": 0.4901,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.5834208726882935,
+      "rewards/margins": 0.8037111163139343,
+      "rewards/rejected": -0.22029033303260803,
+      "step": 50
+    },
+    {
+      "epoch": 0.985781990521327,
+      "eval_logits/chosen": -3.1020591259002686,
+      "eval_logits/rejected": -3.11872935295105,
+      "eval_logps/chosen": -327.0610046386719,
+      "eval_logps/rejected": -340.0373840332031,
+      "eval_loss": 0.5042234063148499,
+      "eval_rewards/accuracies": 0.7708333134651184,
+      "eval_rewards/chosen": 0.4415433406829834,
+      "eval_rewards/margins": 0.8368213772773743,
+      "eval_rewards/rejected": -0.39527806639671326,
+      "eval_runtime": 69.4291,
+      "eval_samples_per_second": 10.802,
+      "eval_steps_per_second": 0.346,
+      "step": 52
+    },
+    {
+      "epoch": 1.1374407582938388,
+      "grad_norm": 23.10640619207419,
+      "learning_rate": 2.2891223348923882e-07,
+      "logits/chosen": -3.099835157394409,
+      "logits/rejected": -3.0689287185668945,
+      "logps/chosen": -351.0329895019531,
+      "logps/rejected": -399.60302734375,
+      "loss": 0.2693,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 1.0113407373428345,
+      "rewards/margins": 1.8710256814956665,
+      "rewards/rejected": -0.8596851229667664,
+      "step": 60
+    },
+    {
+      "epoch": 1.3270142180094786,
+      "grad_norm": 23.257897813967194,
+      "learning_rate": 1.4754491880085317e-07,
+      "logits/chosen": -3.0508038997650146,
+      "logits/rejected": -3.0524604320526123,
+      "logps/chosen": -318.5110778808594,
+      "logps/rejected": -394.3077697753906,
+      "loss": 0.1582,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 1.249901533126831,
+      "rewards/margins": 2.6158621311187744,
+      "rewards/rejected": -1.3659604787826538,
+      "step": 70
+    },
+    {
+      "epoch": 1.5165876777251186,
+      "grad_norm": 20.44465740830546,
+      "learning_rate": 7.775827023107834e-08,
+      "logits/chosen": -3.0571980476379395,
+      "logits/rejected": -3.0500247478485107,
+      "logps/chosen": -358.40460205078125,
+      "logps/rejected": -447.8125,
+      "loss": 0.1456,
+      "rewards/accuracies": 0.981249988079071,
+      "rewards/chosen": 1.2739430665969849,
+      "rewards/margins": 3.01393723487854,
+      "rewards/rejected": -1.7399938106536865,
+      "step": 80
+    },
+    {
+      "epoch": 1.7061611374407581,
+      "grad_norm": 19.260648091922224,
+      "learning_rate": 2.7440387297912122e-08,
+      "logits/chosen": -3.0330233573913574,
+      "logits/rejected": -3.03442645072937,
+      "logps/chosen": -347.0635986328125,
+      "logps/rejected": -439.34326171875,
+      "loss": 0.1318,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 1.133148431777954,
+      "rewards/margins": 3.050485610961914,
+      "rewards/rejected": -1.9173370599746704,
+      "step": 90
+    },
+    {
+      "epoch": 1.8957345971563981,
+      "grad_norm": 19.24195408747522,
+      "learning_rate": 2.27878296044029e-09,
+      "logits/chosen": -3.0184741020202637,
+      "logits/rejected": -3.0368762016296387,
+      "logps/chosen": -332.5245056152344,
+      "logps/rejected": -402.9344177246094,
+      "loss": 0.1326,
+      "rewards/accuracies": 0.9937499761581421,
+      "rewards/chosen": 1.1043305397033691,
+      "rewards/margins": 2.7153286933898926,
+      "rewards/rejected": -1.6109977960586548,
+      "step": 100
+    },
+    {
+      "epoch": 1.971563981042654,
+      "eval_logits/chosen": -3.03796648979187,
+      "eval_logits/rejected": -3.0577399730682373,
+      "eval_logps/chosen": -332.3036804199219,
+      "eval_logps/rejected": -348.6507263183594,
+      "eval_loss": 0.4794084131717682,
+      "eval_rewards/accuracies": 0.7395833134651184,
+      "eval_rewards/chosen": 0.1794087141752243,
+      "eval_rewards/margins": 1.0053516626358032,
+      "eval_rewards/rejected": -0.8259429335594177,
+      "eval_runtime": 69.0646,
+      "eval_samples_per_second": 10.859,
+      "eval_steps_per_second": 0.348,
+      "step": 104
+    },
+    {
+      "epoch": 1.971563981042654,
+      "step": 104,
+      "total_flos": 0.0,
+      "train_loss": 0.3703585032087106,
+      "train_runtime": 3327.4148,
+      "train_samples_per_second": 4.057,
+      "train_steps_per_second": 0.031
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 104,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}