Model save

README.md

@@ -2,16 +2,9 @@
 license: mit
 base_model: HuggingFaceH4/mistral-7b-sft-beta
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
-- trl
-- dpo
-- generated_from_trainer
-datasets:
-- updated
-- original
 model-index:
 - name: 0.001_3iters_bs256_declr_nodpo_userresponse_iter_1
   results: []
@@ -22,7 +15,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # 0.001_3iters_bs256_declr_nodpo_userresponse_iter_1
 
-This model is a fine-tuned version of [HuggingFaceH4/mistral-7b-sft-beta](https://huggingface.co/HuggingFaceH4/mistral-7b-sft-beta) on the updated and the original datasets.
+This model is a fine-tuned version of [HuggingFaceH4/mistral-7b-sft-beta](https://huggingface.co/HuggingFaceH4/mistral-7b-sft-beta) on the None dataset.
 
 ## Model description
 

all_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.9905956112852664,
     "total_flos": 0.0,
-    "train_loss": 0.5763288389278364,
-    "train_runtime": 3299.3537,
+    "train_loss": 0.5773088418984715,
+    "train_runtime": 3291.7593,
     "train_samples": 20378,
-    "train_samples_per_second": 6.176,
+    "train_samples_per_second": 6.191,
     "train_steps_per_second": 0.024
 }

config.json

@@ -21,6 +21,6 @@
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.40.0",
-  "use_cache": true,
+  "use_cache": false,
   "vocab_size": 32000
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ef77c6322332c4c430e9b1ab632749609e42663241e959f8d8662acf03c1b98b
+oid sha256:8f22fca732a9f6b3d6c58ab42eeb476902a58cc9d22a8ae8d11e017da95eda8b
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:219edbf21d323e47afe96ca740d03c559dd91de22d1a308d18b1d386dfd4122c
+oid sha256:8dcc828c3cd23fbfbae0251650a30083fb43da293f1ceec5c3ead5c00a02d75b
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:47c29053b2b9f7324afc708af288407578509665b26aa9b1ef43ca60f52986a8
+oid sha256:7ebcc55be4fad0d9123d62539bc046be1eb9d3d43b4fcd5c7aadd6e2847defb2
 size 4540516344

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.9905956112852664,
     "total_flos": 0.0,
-    "train_loss": 0.5763288389278364,
-    "train_runtime": 3299.3537,
+    "train_loss": 0.5773088418984715,
+    "train_runtime": 3291.7593,
     "train_samples": 20378,
-    "train_samples_per_second": 6.176,
+    "train_samples_per_second": 6.191,
     "train_steps_per_second": 0.024
 }

trainer_state.json

@@ -10,7 +10,7 @@
   "log_history": [
     {
       "epoch": 0.012539184952978056,
-      "grad_norm": 12.995569272950581,
+      "grad_norm": 12.994366765815737,
       "learning_rate": 6.25e-08,
       "logits/chosen": -2.895261764526367,
       "logits/rejected": -2.870915412902832,
@@ -27,130 +27,130 @@
     },
     {
       "epoch": 0.12539184952978055,
-      "grad_norm": 10.597824924824353,
+      "grad_norm": 10.590546723806328,
       "learning_rate": 4.990217055187362e-07,
-      "logits/chosen": -2.7800254821777344,
-      "logits/rejected": -2.764801502227783,
-      "logps/chosen": -215.308349609375,
-      "logps/pi_response": -70.83273315429688,
+      "logits/chosen": -2.779981851577759,
+      "logits/rejected": -2.7648234367370605,
+      "logps/chosen": -215.31658935546875,
+      "logps/pi_response": -70.84928894042969,
       "logps/ref_response": -69.94066619873047,
-      "logps/rejected": -113.73969268798828,
+      "logps/rejected": -113.74195861816406,
       "loss": 0.6874,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.0037591855507344007,
-      "rewards/margins": 0.008248809725046158,
-      "rewards/rejected": -0.004489624407142401,
+      "rewards/accuracies": 0.5659722089767456,
+      "rewards/chosen": 0.003676379332318902,
+      "rewards/margins": 0.008188692852854729,
+      "rewards/rejected": -0.00451231375336647,
       "step": 10
     },
     {
       "epoch": 0.2507836990595611,
-      "grad_norm": 5.532320834011963,
+      "grad_norm": 5.515447952912991,
       "learning_rate": 4.655786431300069e-07,
-      "logits/chosen": -2.677351713180542,
-      "logits/rejected": -2.6526131629943848,
-      "logps/chosen": -242.95101928710938,
-      "logps/pi_response": -88.7496109008789,
+      "logits/chosen": -2.677419900894165,
+      "logits/rejected": -2.6526081562042236,
+      "logps/chosen": -242.970947265625,
+      "logps/pi_response": -88.79570007324219,
       "logps/ref_response": -67.22555541992188,
-      "logps/rejected": -121.909423828125,
-      "loss": 0.6346,
-      "rewards/accuracies": 0.7281249761581421,
-      "rewards/chosen": -0.015267712064087391,
-      "rewards/margins": 0.16114801168441772,
-      "rewards/rejected": -0.17641572654247284,
+      "logps/rejected": -121.95565032958984,
+      "loss": 0.6345,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.015466970391571522,
+      "rewards/margins": 0.16141095757484436,
+      "rewards/rejected": -0.17687790095806122,
       "step": 20
     },
     {
       "epoch": 0.3761755485893417,
-      "grad_norm": 5.696693225944807,
+      "grad_norm": 5.718252994097399,
       "learning_rate": 3.9061232191019517e-07,
-      "logits/chosen": -2.559924602508545,
-      "logits/rejected": -2.529345989227295,
-      "logps/chosen": -235.51962280273438,
-      "logps/pi_response": -122.11323547363281,
+      "logits/chosen": -2.560062885284424,
+      "logits/rejected": -2.529508113861084,
+      "logps/chosen": -235.5645751953125,
+      "logps/pi_response": -122.2088394165039,
       "logps/ref_response": -66.2011489868164,
-      "logps/rejected": -152.02407836914062,
-      "loss": 0.5998,
+      "logps/rejected": -152.08804321289062,
+      "loss": 0.5997,
       "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.19383877515792847,
-      "rewards/margins": 0.27510958909988403,
-      "rewards/rejected": -0.4689483046531677,
+      "rewards/chosen": -0.1942879855632782,
+      "rewards/margins": 0.27530020475387573,
+      "rewards/rejected": -0.4695882201194763,
       "step": 30
     },
     {
       "epoch": 0.5015673981191222,
-      "grad_norm": 7.1826029149350745,
+      "grad_norm": 6.9513019055279806,
       "learning_rate": 2.8856223324132555e-07,
-      "logits/chosen": -2.5519003868103027,
-      "logits/rejected": -2.5247206687927246,
-      "logps/chosen": -251.58963012695312,
-      "logps/pi_response": -164.69174194335938,
+      "logits/chosen": -2.5515570640563965,
+      "logits/rejected": -2.5246214866638184,
+      "logps/chosen": -250.9296112060547,
+      "logps/pi_response": -163.28016662597656,
       "logps/ref_response": -70.73230743408203,
-      "logps/rejected": -184.05503845214844,
-      "loss": 0.5645,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.3932897448539734,
-      "rewards/margins": 0.4179116189479828,
-      "rewards/rejected": -0.8112013936042786,
+      "logps/rejected": -182.60157775878906,
+      "loss": 0.5653,
+      "rewards/accuracies": 0.753125011920929,
+      "rewards/chosen": -0.38668954372406006,
+      "rewards/margins": 0.40997713804244995,
+      "rewards/rejected": -0.7966667413711548,
       "step": 40
     },
     {
       "epoch": 0.6269592476489029,
-      "grad_norm": 8.887185971495962,
+      "grad_norm": 9.300006842094811,
       "learning_rate": 1.7908455541642582e-07,
-      "logits/chosen": -2.558701753616333,
-      "logits/rejected": -2.52125883102417,
-      "logps/chosen": -286.60028076171875,
-      "logps/pi_response": -173.02108764648438,
+      "logits/chosen": -2.5594820976257324,
+      "logits/rejected": -2.5220818519592285,
+      "logps/chosen": -286.2401428222656,
+      "logps/pi_response": -171.46835327148438,
       "logps/ref_response": -68.65879821777344,
-      "logps/rejected": -191.91122436523438,
-      "loss": 0.5437,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -0.38747724890708923,
-      "rewards/margins": 0.5338504314422607,
-      "rewards/rejected": -0.9213277101516724,
+      "logps/rejected": -190.53627014160156,
+      "loss": 0.5449,
+      "rewards/accuracies": 0.784375011920929,
+      "rewards/chosen": -0.3838757574558258,
+      "rewards/margins": 0.5237023234367371,
+      "rewards/rejected": -0.90757817029953,
       "step": 50
     },
     {
       "epoch": 0.7523510971786834,
-      "grad_norm": 16.631565802707648,
+      "grad_norm": 16.372680174050775,
       "learning_rate": 8.32661172908373e-08,
-      "logits/chosen": -2.553363800048828,
-      "logits/rejected": -2.5198798179626465,
-      "logps/chosen": -244.05642700195312,
-      "logps/pi_response": -171.74761962890625,
+      "logits/chosen": -2.552124261856079,
+      "logits/rejected": -2.518066167831421,
+      "logps/chosen": -246.05886840820312,
+      "logps/pi_response": -173.25637817382812,
       "logps/ref_response": -62.150115966796875,
-      "logps/rejected": -194.5513458251953,
-      "loss": 0.536,
+      "logps/rejected": -196.02783203125,
+      "loss": 0.5395,
       "rewards/accuracies": 0.7593749761581421,
-      "rewards/chosen": -0.41183900833129883,
-      "rewards/margins": 0.5683741569519043,
-      "rewards/rejected": -0.9802130460739136,
+      "rewards/chosen": -0.4318634569644928,
+      "rewards/margins": 0.563114583492279,
+      "rewards/rejected": -0.9949780702590942,
       "step": 60
     },
     {
       "epoch": 0.877742946708464,
-      "grad_norm": 10.81361845883075,
+      "grad_norm": 8.403502856566952,
       "learning_rate": 1.956279997278043e-08,
-      "logits/chosen": -2.5614013671875,
-      "logits/rejected": -2.529111385345459,
-      "logps/chosen": -282.7896423339844,
-      "logps/pi_response": -187.90109252929688,
+      "logits/chosen": -2.559234619140625,
+      "logits/rejected": -2.527210235595703,
+      "logps/chosen": -281.3694763183594,
+      "logps/pi_response": -184.9126739501953,
       "logps/ref_response": -70.37203216552734,
-      "logps/rejected": -210.452392578125,
-      "loss": 0.5313,
-      "rewards/accuracies": 0.753125011920929,
-      "rewards/chosen": -0.42874473333358765,
-      "rewards/margins": 0.621096134185791,
-      "rewards/rejected": -1.0498409271240234,
+      "logps/rejected": -207.85086059570312,
+      "loss": 0.5331,
+      "rewards/accuracies": 0.7406250238418579,
+      "rewards/chosen": -0.4145432412624359,
+      "rewards/margins": 0.6092821359634399,
+      "rewards/rejected": -1.0238254070281982,
       "step": 70
     },
     {
       "epoch": 0.9905956112852664,
       "step": 79,
       "total_flos": 0.0,
-      "train_loss": 0.5763288389278364,
-      "train_runtime": 3299.3537,
-      "train_samples_per_second": 6.176,
+      "train_loss": 0.5773088418984715,
+      "train_runtime": 3291.7593,
+      "train_samples_per_second": 6.191,
       "train_steps_per_second": 0.024
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4d0c3e76061c137e90866e02aa6bbc07931f10e4c35de081b18dec8af745d317
+oid sha256:c739fd0ead78f45a1f03fbec91a53ef8de7c93f38163455a579aedbbfb77d02b
 size 6328