Model save

README.md

@@ -16,15 +16,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model was trained from scratch on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.4802
-- Rewards/chosen: 0.1868
-- Rewards/rejected: -0.8036
-- Rewards/accuracies: 0.75
-- Rewards/margins: 0.9904
-- Logps/rejected: -347.3108
-- Logps/chosen: -331.7999
-- Logits/rejected: -3.0382
-- Logits/chosen: -3.0187
+- Loss: 0.5490
+- Rewards/chosen: -0.3865
+- Rewards/rejected: -1.0064
+- Rewards/accuracies: 0.6915
+- Rewards/margins: 0.6199
+- Logps/rejected: -415.0506
+- Logps/chosen: -355.7762
+- Logits/rejected: 326.4959
+- Logits/chosen: 326.0051
 
 ## Model description
 
@@ -44,14 +44,14 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 5e-07
-- train_batch_size: 4
-- eval_batch_size: 4
+- train_batch_size: 1
+- eval_batch_size: 2
 - seed: 42
 - distributed_type: multi-GPU
 - num_devices: 8
-- gradient_accumulation_steps: 4
+- gradient_accumulation_steps: 16
 - total_train_batch_size: 128
-- total_eval_batch_size: 32
+- total_eval_batch_size: 16
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_ratio: 0.1
@@ -61,8 +61,8 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.493         | 0.9858 | 52   | 0.5064          | 0.4437         | -0.3824          | 0.7812             | 0.8261          | -338.8869      | -326.6608    | -3.1043         | -3.0877       |
-| 0.1356        | 1.9716 | 104  | 0.4802          | 0.1868         | -0.8036          | 0.75               | 0.9904          | -347.3108      | -331.7999    | -3.0382         | -3.0187       |
+| 0.5334        | 0.9858 | 52   | 0.5680          | -0.2471        | -0.7713          | 0.6702             | 0.5242          | -410.3489      | -352.9877    | 326.9832        | 326.6964      |
+| 0.2495        | 1.9716 | 104  | 0.5490          | -0.3865        | -1.0064          | 0.6915             | 0.6199          | -415.0506      | -355.7762    | 326.4959        | 326.0051      |
 
 
 ### Framework versions

all_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 1.971563981042654,
     "total_flos": 0.0,
-    "train_loss": 0.373156875371933,
-    "train_runtime": 3298.3137,
+    "train_loss": 0.44703544962864655,
+    "train_runtime": 4801.7476,
     "train_samples": 6750,
-    "train_samples_per_second": 4.093,
-    "train_steps_per_second": 0.032
+    "train_samples_per_second": 2.811,
+    "train_steps_per_second": 0.022
 }

generation_config.json

@@ -1,6 +1,7 @@
 {
   "_from_model_config": true,
-  "bos_token_id": 1,
-  "eos_token_id": 2,
+  "bos_token_id": 2,
+  "eos_token_id": 1,
+  "pad_token_id": 0,
   "transformers_version": "4.45.2"
 }

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 1.971563981042654,
     "total_flos": 0.0,
-    "train_loss": 0.373156875371933,
-    "train_runtime": 3298.3137,
+    "train_loss": 0.44703544962864655,
+    "train_runtime": 4801.7476,
     "train_samples": 6750,
-    "train_samples_per_second": 4.093,
-    "train_steps_per_second": 0.032
+    "train_samples_per_second": 2.811,
+    "train_steps_per_second": 0.022
 }

trainer_state.json

@@ -10,12 +10,12 @@
   "log_history": [
     {
       "epoch": 0.018957345971563982,
-      "grad_norm": 83.90713111595433,
+      "grad_norm": 64.29333218701703,
       "learning_rate": 4.545454545454545e-08,
-      "logits/chosen": -2.873765468597412,
-      "logits/rejected": -2.883460521697998,
-      "logps/chosen": -422.4774169921875,
-      "logps/rejected": -480.524169921875,
+      "logits/chosen": 260.044921875,
+      "logits/rejected": 273.92498779296875,
+      "logps/chosen": -421.27789306640625,
+      "logps/rejected": -408.871337890625,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -25,194 +25,194 @@
     },
     {
       "epoch": 0.1895734597156398,
-      "grad_norm": 77.5065062501301,
+      "grad_norm": 66.1506989600867,
       "learning_rate": 4.545454545454545e-07,
-      "logits/chosen": -2.859947681427002,
-      "logits/rejected": -2.877683162689209,
-      "logps/chosen": -346.8866882324219,
-      "logps/rejected": -398.614013671875,
-      "loss": 0.6895,
-      "rewards/accuracies": 0.4930555522441864,
-      "rewards/chosen": 0.007129705511033535,
-      "rewards/margins": 0.008572477847337723,
-      "rewards/rejected": -0.001442772219888866,
+      "logits/chosen": 282.578857421875,
+      "logits/rejected": 277.1462097167969,
+      "logps/chosen": -362.7806091308594,
+      "logps/rejected": -423.041015625,
+      "loss": 0.7032,
+      "rewards/accuracies": 0.5347222089767456,
+      "rewards/chosen": 0.04673011228442192,
+      "rewards/margins": 0.09045842289924622,
+      "rewards/rejected": -0.043728310614824295,
       "step": 10
     },
     {
       "epoch": 0.3791469194312796,
-      "grad_norm": 68.52599234402585,
+      "grad_norm": 69.27857567048012,
       "learning_rate": 4.885348141000122e-07,
-      "logits/chosen": -2.9137072563171387,
-      "logits/rejected": -2.92988920211792,
-      "logps/chosen": -389.4097595214844,
-      "logps/rejected": -417.3075256347656,
-      "loss": 0.643,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.13587789237499237,
-      "rewards/margins": 0.10481055080890656,
-      "rewards/rejected": 0.031067365780472755,
+      "logits/chosen": 276.78228759765625,
+      "logits/rejected": 275.58184814453125,
+      "logps/chosen": -364.314208984375,
+      "logps/rejected": -391.9237365722656,
+      "loss": 0.6778,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": 0.04571449011564255,
+      "rewards/margins": 0.1900663673877716,
+      "rewards/rejected": -0.14435191452503204,
       "step": 20
     },
     {
       "epoch": 0.5687203791469194,
-      "grad_norm": 61.72689387884387,
+      "grad_norm": 56.71129503742715,
       "learning_rate": 4.5025027361734613e-07,
-      "logits/chosen": -2.9870388507843018,
-      "logits/rejected": -2.9750781059265137,
-      "logps/chosen": -334.49029541015625,
-      "logps/rejected": -352.12322998046875,
-      "loss": 0.5547,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.3548569083213806,
-      "rewards/margins": 0.3839985430240631,
-      "rewards/rejected": -0.02914164587855339,
+      "logits/chosen": 276.2879333496094,
+      "logits/rejected": 274.1214904785156,
+      "logps/chosen": -326.874755859375,
+      "logps/rejected": -380.1121520996094,
+      "loss": 0.6173,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.22750525176525116,
+      "rewards/margins": 0.4899619221687317,
+      "rewards/rejected": -0.7174672484397888,
       "step": 30
     },
     {
       "epoch": 0.7582938388625592,
-      "grad_norm": 64.8854081169712,
+      "grad_norm": 57.871031614943476,
       "learning_rate": 3.893311157806091e-07,
-      "logits/chosen": -3.0583784580230713,
-      "logits/rejected": -3.054187536239624,
-      "logps/chosen": -364.53973388671875,
-      "logps/rejected": -349.81390380859375,
-      "loss": 0.5569,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": 0.515569269657135,
-      "rewards/margins": 0.7298178672790527,
-      "rewards/rejected": -0.2142486274242401,
+      "logits/chosen": 277.44049072265625,
+      "logits/rejected": 286.52374267578125,
+      "logps/chosen": -351.580078125,
+      "logps/rejected": -366.3487854003906,
+      "loss": 0.5947,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.05702618509531021,
+      "rewards/margins": 0.7331444025039673,
+      "rewards/rejected": -0.790170431137085,
       "step": 40
     },
     {
       "epoch": 0.9478672985781991,
-      "grad_norm": 64.27580122584432,
+      "grad_norm": 73.77852490676803,
       "learning_rate": 3.126631330646801e-07,
-      "logits/chosen": -3.055851459503174,
-      "logits/rejected": -3.055706739425659,
-      "logps/chosen": -368.67901611328125,
-      "logps/rejected": -415.5660095214844,
-      "loss": 0.493,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.567669153213501,
-      "rewards/margins": 0.7747016549110413,
-      "rewards/rejected": -0.2070324867963791,
+      "logits/chosen": 275.7063293457031,
+      "logits/rejected": 279.91253662109375,
+      "logps/chosen": -378.2023010253906,
+      "logps/rejected": -397.1396179199219,
+      "loss": 0.5334,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.07171835005283356,
+      "rewards/margins": 0.9270683526992798,
+      "rewards/rejected": -0.9987867474555969,
       "step": 50
     },
     {
       "epoch": 0.985781990521327,
-      "eval_logits/chosen": -3.087723970413208,
-      "eval_logits/rejected": -3.104304313659668,
-      "eval_logps/chosen": -326.6607666015625,
-      "eval_logps/rejected": -338.8869323730469,
-      "eval_loss": 0.5064198970794678,
-      "eval_rewards/accuracies": 0.78125,
-      "eval_rewards/chosen": 0.44371843338012695,
-      "eval_rewards/margins": 0.8261151313781738,
-      "eval_rewards/rejected": -0.3823966085910797,
-      "eval_runtime": 69.4367,
-      "eval_samples_per_second": 10.801,
-      "eval_steps_per_second": 0.346,
+      "eval_logits/chosen": 326.6963806152344,
+      "eval_logits/rejected": 326.98321533203125,
+      "eval_logps/chosen": -352.98773193359375,
+      "eval_logps/rejected": -410.34893798828125,
+      "eval_loss": 0.5679783821105957,
+      "eval_rewards/accuracies": 0.6702127456665039,
+      "eval_rewards/chosen": -0.2470909059047699,
+      "eval_rewards/margins": 0.52419114112854,
+      "eval_rewards/rejected": -0.7712819576263428,
+      "eval_runtime": 90.9704,
+      "eval_samples_per_second": 8.244,
+      "eval_steps_per_second": 0.517,
       "step": 52
     },
     {
       "epoch": 1.1374407582938388,
-      "grad_norm": 23.801523175944517,
+      "grad_norm": 31.05292858682528,
       "learning_rate": 2.2891223348923882e-07,
-      "logits/chosen": -3.084404945373535,
-      "logits/rejected": -3.054302930831909,
-      "logps/chosen": -351.47882080078125,
-      "logps/rejected": -398.0810852050781,
-      "loss": 0.2739,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.9971694946289062,
-      "rewards/margins": 1.8351186513900757,
-      "rewards/rejected": -0.8379490971565247,
+      "logits/chosen": 274.8268737792969,
+      "logits/rejected": 274.29949951171875,
+      "logps/chosen": -326.9039611816406,
+      "logps/rejected": -397.5994567871094,
+      "loss": 0.3923,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.19279229640960693,
+      "rewards/margins": 1.5808745622634888,
+      "rewards/rejected": -1.3880822658538818,
       "step": 60
     },
     {
       "epoch": 1.3270142180094786,
-      "grad_norm": 23.77088500982387,
+      "grad_norm": 33.93405312466621,
       "learning_rate": 1.4754491880085317e-07,
-      "logits/chosen": -3.0304384231567383,
-      "logits/rejected": -3.0326294898986816,
-      "logps/chosen": -318.4073791503906,
-      "logps/rejected": -392.42071533203125,
-      "loss": 0.1625,
-      "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": 1.2244815826416016,
-      "rewards/margins": 2.5673928260803223,
-      "rewards/rejected": -1.3429110050201416,
+      "logits/chosen": 279.77435302734375,
+      "logits/rejected": 278.4494934082031,
+      "logps/chosen": -331.9073486328125,
+      "logps/rejected": -370.1907653808594,
+      "loss": 0.2876,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 0.4298977851867676,
+      "rewards/margins": 2.020857334136963,
+      "rewards/rejected": -1.5909594297409058,
       "step": 70
     },
     {
       "epoch": 1.5165876777251186,
-      "grad_norm": 19.870466912542945,
+      "grad_norm": 23.235180840728248,
       "learning_rate": 7.775827023107834e-08,
-      "logits/chosen": -3.0366415977478027,
-      "logits/rejected": -3.0298948287963867,
-      "logps/chosen": -358.39984130859375,
-      "logps/rejected": -446.78857421875,
-      "loss": 0.1491,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 1.2719032764434814,
-      "rewards/margins": 2.9869067668914795,
-      "rewards/rejected": -1.7150036096572876,
+      "logits/chosen": 281.94732666015625,
+      "logits/rejected": 273.55474853515625,
+      "logps/chosen": -365.9495849609375,
+      "logps/rejected": -429.3880310058594,
+      "loss": 0.2574,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.3758580982685089,
+      "rewards/margins": 2.054001569747925,
+      "rewards/rejected": -1.6781437397003174,
       "step": 80
     },
     {
       "epoch": 1.7061611374407581,
-      "grad_norm": 19.556816629836018,
+      "grad_norm": 26.571064677269206,
       "learning_rate": 2.7440387297912122e-08,
-      "logits/chosen": -3.0141148567199707,
-      "logits/rejected": -3.015047073364258,
-      "logps/chosen": -346.9141540527344,
-      "logps/rejected": -437.4671325683594,
-      "loss": 0.1358,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/chosen": 1.1179364919662476,
-      "rewards/margins": 2.981201410293579,
-      "rewards/rejected": -1.863265037536621,
+      "logits/chosen": 276.1026916503906,
+      "logits/rejected": 270.5816345214844,
+      "logps/chosen": -347.1163330078125,
+      "logps/rejected": -437.5128479003906,
+      "loss": 0.2439,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 0.39976122975349426,
+      "rewards/margins": 2.205505847930908,
+      "rewards/rejected": -1.8057447671890259,
       "step": 90
     },
     {
       "epoch": 1.8957345971563981,
-      "grad_norm": 18.965988787956388,
+      "grad_norm": 25.094637702590482,
       "learning_rate": 2.27878296044029e-09,
-      "logits/chosen": -2.9989800453186035,
-      "logits/rejected": -3.0166327953338623,
-      "logps/chosen": -332.5897216796875,
-      "logps/rejected": -401.76165771484375,
-      "loss": 0.1356,
-      "rewards/accuracies": 0.9937499761581421,
-      "rewards/chosen": 1.0875240564346313,
-      "rewards/margins": 2.662875175476074,
-      "rewards/rejected": -1.5753511190414429,
+      "logits/chosen": 278.7438659667969,
+      "logits/rejected": 279.1758728027344,
+      "logps/chosen": -350.7715148925781,
+      "logps/rejected": -397.90484619140625,
+      "loss": 0.2495,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 0.19362537562847137,
+      "rewards/margins": 2.171940565109253,
+      "rewards/rejected": -1.9783153533935547,
       "step": 100
     },
     {
       "epoch": 1.971563981042654,
-      "eval_logits/chosen": -3.0187489986419678,
-      "eval_logits/rejected": -3.038198232650757,
-      "eval_logps/chosen": -331.7999267578125,
-      "eval_logps/rejected": -347.3108215332031,
-      "eval_loss": 0.48019182682037354,
-      "eval_rewards/accuracies": 0.75,
-      "eval_rewards/chosen": 0.18675994873046875,
-      "eval_rewards/margins": 0.9903542399406433,
-      "eval_rewards/rejected": -0.803594172000885,
-      "eval_runtime": 69.326,
-      "eval_samples_per_second": 10.818,
-      "eval_steps_per_second": 0.346,
+      "eval_logits/chosen": 326.0050964355469,
+      "eval_logits/rejected": 326.4959411621094,
+      "eval_logps/chosen": -355.77618408203125,
+      "eval_logps/rejected": -415.0506286621094,
+      "eval_loss": 0.548955500125885,
+      "eval_rewards/accuracies": 0.6914893388748169,
+      "eval_rewards/chosen": -0.38651418685913086,
+      "eval_rewards/margins": 0.6198524832725525,
+      "eval_rewards/rejected": -1.0063666105270386,
+      "eval_runtime": 87.0456,
+      "eval_samples_per_second": 8.616,
+      "eval_steps_per_second": 0.54,
       "step": 104
     },
     {
       "epoch": 1.971563981042654,
       "step": 104,
       "total_flos": 0.0,
-      "train_loss": 0.373156875371933,
-      "train_runtime": 3298.3137,
-      "train_samples_per_second": 4.093,
-      "train_steps_per_second": 0.032
+      "train_loss": 0.44703544962864655,
+      "train_runtime": 4801.7476,
+      "train_samples_per_second": 2.811,
+      "train_steps_per_second": 0.022
     }
   ],
   "logging_steps": 10,
@@ -233,7 +233,7 @@
     }
   },
   "total_flos": 0.0,
-  "train_batch_size": 4,
+  "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null
 }