Model save

README.md

@@ -1,9 +1,7 @@
 ---
 base_model: Sim4Rec/inter-play-sim-assistant-sft
-datasets:
-- Sim4Rec/dpo_data
 library_name: transformers
-model_name: Sim4Rec/inter-play-sim-assistant-sft
+model_name: inter-play-sim-assistant-dpo
 tags:
 - generated_from_trainer
 - trl
@@ -11,9 +9,9 @@ tags:
 licence: license
 ---
 
-# Model Card for Sim4Rec/inter-play-sim-assistant-sft
+# Model Card for inter-play-sim-assistant-dpo
 
-This model is a fine-tuned version of [Sim4Rec/inter-play-sim-assistant-sft](https://huggingface.co/Sim4Rec/inter-play-sim-assistant-sft) on the [['Sim4Rec/dpo_data']](https://huggingface.co/datasets/['Sim4Rec/dpo_data']) dataset.
+This model is a fine-tuned version of [Sim4Rec/inter-play-sim-assistant-sft](https://huggingface.co/Sim4Rec/inter-play-sim-assistant-sft).
 It has been trained using [TRL](https://github.com/huggingface/trl).
 
 ## Quick start
@@ -29,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/jerome-ramos-20/huggingface/runs/q3zfrxdb) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/jerome-ramos-20/huggingface/runs/k9xn3f7n) 
 
 
 This model was trained with DPO, a method introduced in [Direct Preference Optimization: Your Language Model is Secretly a Reward Model](https://huggingface.co/papers/2305.18290).

all_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 2.0,
+    "epoch": 1.0,
     "total_flos": 0.0,
-    "train_loss": 0.22515048312672067,
-    "train_runtime": 2958.2669,
+    "train_loss": 0.23233672156092827,
+    "train_runtime": 2773.067,
     "train_samples": 45561,
-    "train_samples_per_second": 30.802,
-    "train_steps_per_second": 0.241
+    "train_samples_per_second": 16.43,
+    "train_steps_per_second": 0.257
 }

config.json

@@ -32,6 +32,6 @@
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.48.2",
-  "use_cache": true,
+  "use_cache": false,
   "vocab_size": 128320
 }

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:228fda64e183c9fdc3a7708fb3208210cdadad23990e8923b3fe182ecf5c0cec
+oid sha256:b1dca7caf67b0af32b1aa5c9c52ad02de5b54e84a8ec038bd74e3bc91b18d8c7
 size 4977222960

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5f7204be8a0fc4f037c035587cfd68a41793f7158bf6cf5794a16e4fe64cbc8a
+oid sha256:7f09aa5fdc5b3591ad4b03edba26dce3703417eba2a78ff25e3f469b3664a254
 size 4999802720

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:48b841d9240b04b4dfe85f97b3b566448cfbabe276eac748a7ad65411df8577f
+oid sha256:da6625e7a5cfcdd77627f26512cf498048fc24b6547e71e3ef4c3a71e41141d9
 size 4915916176

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2c785dc0e0713c891a93b159479d9de5baf67a041e01652bf3e46d23a5251c8c
+oid sha256:6115a28a38e67e9c28cd5db2742d2794509bfac5395d3a4d4f58a995d926ed67
 size 1168663096

runs/Feb17_14-09-57_w-jerom-inter-play-sim-94c6890b9ccf44ea86f033a3db8a5dbd-6d4ql9c/events.out.tfevents.1739801633.w-jerom-inter-play-sim-94c6890b9ccf44ea86f033a3db8a5dbd-6d4ql9c.17212.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e9be616f76489d27fc902d791a648442cd26dc2c39ccc9b948efb7c34c0330f
+size 56172

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3919c1e7bfa558ff525a618a3d463929a238acaba668d7ef6da432fcd6cd7fad
-size 17211327
+oid sha256:b5ea5afcc70a5f73f9b545a5940b211fd23e2acd4d895a3ebc3144ca348a4633
+size 17211228

train_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 2.0,
+    "epoch": 1.0,
     "total_flos": 0.0,
-    "train_loss": 0.22515048312672067,
-    "train_runtime": 2958.2669,
+    "train_loss": 0.23233672156092827,
+    "train_runtime": 2773.067,
     "train_samples": 45561,
-    "train_samples_per_second": 30.802,
-    "train_steps_per_second": 0.241
+    "train_samples_per_second": 16.43,
+    "train_steps_per_second": 0.257
 }

trainer_state.json

@@ -1,7 +1,7 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 2.0,
+  "epoch": 1.0,
   "eval_steps": 500,
   "global_step": 712,
   "is_hyper_param_search": false,
@@ -9,13 +9,13 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.0028089887640449437,
-      "grad_norm": 1.1993484497070312,
+      "epoch": 0.0014044943820224719,
+      "grad_norm": 0.9453608393669128,
       "learning_rate": 6.9444444444444435e-09,
-      "logits/chosen": -3.220703125,
-      "logits/rejected": -3.1796875,
-      "logps/chosen": -43.75,
-      "logps/rejected": -42.78125,
+      "logits/chosen": -3.205078125,
+      "logits/rejected": -3.185546875,
+      "logps/chosen": -43.59375,
+      "logps/rejected": -42.640625,
       "loss": 0.6914,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -24,1084 +24,1084 @@
       "step": 1
     },
     {
-      "epoch": 0.028089887640449437,
-      "grad_norm": 0.7986037135124207,
+      "epoch": 0.014044943820224719,
+      "grad_norm": 1.151463508605957,
       "learning_rate": 6.944444444444444e-08,
-      "logits/chosen": -3.2486979961395264,
-      "logits/rejected": -3.2052950859069824,
-      "logps/chosen": -42.34375,
-      "logps/rejected": -41.01215362548828,
-      "loss": 0.6921,
-      "rewards/accuracies": 0.2161458283662796,
-      "rewards/chosen": 0.00020620558643713593,
-      "rewards/margins": -0.00015253490710165352,
-      "rewards/rejected": 0.0003587139945011586,
+      "logits/chosen": -3.24609375,
+      "logits/rejected": -3.196831703186035,
+      "logps/chosen": -42.70138931274414,
+      "logps/rejected": -41.57638931274414,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.2465277761220932,
+      "rewards/chosen": 9.75396906142123e-05,
+      "rewards/margins": -0.00013128916907589883,
+      "rewards/rejected": 0.00022856394934933633,
       "step": 10
     },
     {
-      "epoch": 0.056179775280898875,
-      "grad_norm": 0.6845986247062683,
+      "epoch": 0.028089887640449437,
+      "grad_norm": 0.5042410492897034,
       "learning_rate": 1.3888888888888888e-07,
-      "logits/chosen": -3.244140625,
-      "logits/rejected": -3.219531297683716,
-      "logps/chosen": -41.37812423706055,
-      "logps/rejected": -40.42656326293945,
-      "loss": 0.6907,
-      "rewards/accuracies": 0.3460937440395355,
-      "rewards/chosen": 0.006803750991821289,
-      "rewards/margins": 0.003222918603569269,
-      "rewards/rejected": 0.003580451011657715,
+      "logits/chosen": -3.255859375,
+      "logits/rejected": -3.21484375,
+      "logps/chosen": -42.06718826293945,
+      "logps/rejected": -40.62968826293945,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.30937498807907104,
+      "rewards/chosen": 0.006805038545280695,
+      "rewards/margins": 0.0023165703751146793,
+      "rewards/rejected": 0.004488563630729914,
       "step": 20
     },
     {
-      "epoch": 0.08426966292134831,
-      "grad_norm": 0.5183084607124329,
+      "epoch": 0.042134831460674156,
+      "grad_norm": 0.6343653202056885,
       "learning_rate": 2.0833333333333333e-07,
-      "logits/chosen": -3.263671875,
-      "logits/rejected": -3.2261719703674316,
-      "logps/chosen": -41.142189025878906,
-      "logps/rejected": -40.00312423706055,
-      "loss": 0.6862,
-      "rewards/accuracies": 0.44140625,
-      "rewards/chosen": 0.02759246900677681,
-      "rewards/margins": 0.013646435923874378,
-      "rewards/rejected": 0.013950538821518421,
+      "logits/chosen": -3.262500047683716,
+      "logits/rejected": -3.232617139816284,
+      "logps/chosen": -41.40625,
+      "logps/rejected": -40.610939025878906,
+      "loss": 0.6869,
+      "rewards/accuracies": 0.4468750059604645,
+      "rewards/chosen": 0.027604103088378906,
+      "rewards/margins": 0.012437248602509499,
+      "rewards/rejected": 0.015170956030488014,
       "step": 30
     },
     {
-      "epoch": 0.11235955056179775,
-      "grad_norm": 0.5848017334938049,
+      "epoch": 0.056179775280898875,
+      "grad_norm": 0.7174843549728394,
       "learning_rate": 2.7777777777777776e-07,
-      "logits/chosen": -3.286328077316284,
-      "logits/rejected": -3.229296922683716,
-      "logps/chosen": -41.439064025878906,
-      "logps/rejected": -41.0,
-      "loss": 0.6745,
-      "rewards/accuracies": 0.5546875,
-      "rewards/chosen": 0.039928339421749115,
-      "rewards/margins": 0.038716744631528854,
-      "rewards/rejected": 0.001224255538545549,
+      "logits/chosen": -3.279296875,
+      "logits/rejected": -3.2447266578674316,
+      "logps/chosen": -40.29218673706055,
+      "logps/rejected": -39.842185974121094,
+      "loss": 0.68,
+      "rewards/accuracies": 0.5078125,
+      "rewards/chosen": 0.03814506530761719,
+      "rewards/margins": 0.027390670031309128,
+      "rewards/rejected": 0.010743332095444202,
       "step": 40
     },
     {
-      "epoch": 0.1404494382022472,
-      "grad_norm": 0.7922030091285706,
+      "epoch": 0.0702247191011236,
+      "grad_norm": 0.5838291645050049,
       "learning_rate": 3.472222222222222e-07,
-      "logits/chosen": -3.2728514671325684,
-      "logits/rejected": NaN,
-      "logps/chosen": -41.71406173706055,
-      "logps/rejected": -41.890625,
-      "loss": 0.6535,
-      "rewards/accuracies": 0.645312488079071,
-      "rewards/chosen": 0.03761863708496094,
-      "rewards/margins": 0.08368835598230362,
-      "rewards/rejected": -0.04610452800989151,
+      "logits/chosen": -3.2763671875,
+      "logits/rejected": -3.230664014816284,
+      "logps/chosen": -40.84687423706055,
+      "logps/rejected": -40.31562423706055,
+      "loss": 0.6619,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": 0.041875459253787994,
+      "rewards/margins": 0.06587791442871094,
+      "rewards/rejected": -0.023966407403349876,
       "step": 50
     },
     {
-      "epoch": 0.16853932584269662,
-      "grad_norm": 1.3905400037765503,
+      "epoch": 0.08426966292134831,
+      "grad_norm": 0.8917225003242493,
       "learning_rate": 4.1666666666666667e-07,
-      "logits/chosen": -3.204296827316284,
-      "logits/rejected": -3.154296875,
-      "logps/chosen": -44.493751525878906,
-      "logps/rejected": -48.446876525878906,
-      "loss": 0.5839,
-      "rewards/accuracies": 0.692187488079071,
-      "rewards/chosen": -0.11696071922779083,
-      "rewards/margins": 0.25516968965530396,
-      "rewards/rejected": -0.3720703125,
+      "logits/chosen": -3.2505860328674316,
+      "logits/rejected": -3.1898436546325684,
+      "logps/chosen": -41.443748474121094,
+      "logps/rejected": -43.939064025878906,
+      "loss": 0.6155,
+      "rewards/accuracies": 0.6890624761581421,
+      "rewards/chosen": 0.011277198791503906,
+      "rewards/margins": 0.17209243774414062,
+      "rewards/rejected": -0.1607826203107834,
       "step": 60
     },
     {
-      "epoch": 0.19662921348314608,
-      "grad_norm": 0.8105548024177551,
+      "epoch": 0.09831460674157304,
+      "grad_norm": 1.416306734085083,
       "learning_rate": 4.861111111111111e-07,
-      "logits/chosen": -3.23828125,
-      "logits/rejected": -3.130859375,
-      "logps/chosen": -47.71875,
-      "logps/rejected": -59.140625,
-      "loss": 0.4754,
-      "rewards/accuracies": 0.7093750238418579,
-      "rewards/chosen": -0.2639709413051605,
-      "rewards/margins": 0.6321045160293579,
-      "rewards/rejected": -0.895751953125,
+      "logits/chosen": -3.2109375,
+      "logits/rejected": -3.1263670921325684,
+      "logps/chosen": -46.9375,
+      "logps/rejected": -53.76874923706055,
+      "loss": 0.5336,
+      "rewards/accuracies": 0.7015625238418579,
+      "rewards/chosen": -0.21380920708179474,
+      "rewards/margins": 0.407858282327652,
+      "rewards/rejected": -0.6218963861465454,
       "step": 70
     },
     {
-      "epoch": 0.2247191011235955,
-      "grad_norm": 0.6645036339759827,
+      "epoch": 0.11235955056179775,
+      "grad_norm": 0.6724388599395752,
       "learning_rate": 4.998072590601808e-07,
-      "logits/chosen": -3.2529296875,
-      "logits/rejected": NaN,
-      "logps/chosen": -43.765625,
-      "logps/rejected": -72.40937805175781,
-      "loss": 0.3394,
-      "rewards/accuracies": 0.73828125,
-      "rewards/chosen": -0.08822021633386612,
-      "rewards/margins": 1.5029785633087158,
-      "rewards/rejected": -1.591284155845642,
+      "logits/chosen": -3.2593750953674316,
+      "logits/rejected": -3.081835985183716,
+      "logps/chosen": -44.157814025878906,
+      "logps/rejected": -63.279685974121094,
+      "loss": 0.405,
+      "rewards/accuracies": 0.7203124761581421,
+      "rewards/chosen": -0.117925263941288,
+      "rewards/margins": 1.0123169422149658,
+      "rewards/rejected": -1.1299316883087158,
       "step": 80
     },
     {
-      "epoch": 0.25280898876404495,
-      "grad_norm": 0.436211496591568,
+      "epoch": 0.12640449438202248,
+      "grad_norm": 0.8954005837440491,
       "learning_rate": 4.990247583129217e-07,
-      "logits/chosen": -3.268749952316284,
-      "logits/rejected": -2.9375,
-      "logps/chosen": -46.046875,
-      "logps/rejected": -89.46875,
-      "loss": 0.2745,
-      "rewards/accuracies": 0.7476562261581421,
-      "rewards/chosen": -0.22807636857032776,
-      "rewards/margins": 2.234375,
-      "rewards/rejected": -2.463183641433716,
+      "logits/chosen": -3.230273485183716,
+      "logits/rejected": -2.9839844703674316,
+      "logps/chosen": -45.54218673706055,
+      "logps/rejected": -78.46562194824219,
+      "loss": 0.3109,
+      "rewards/accuracies": 0.7515624761581421,
+      "rewards/chosen": -0.15528163313865662,
+      "rewards/margins": 1.7075684070587158,
+      "rewards/rejected": -1.863037109375,
       "step": 90
     },
     {
-      "epoch": 0.2808988764044944,
-      "grad_norm": 0.43823757767677307,
+      "epoch": 0.1404494382022472,
+      "grad_norm": 0.5155877470970154,
       "learning_rate": 4.976423351108942e-07,
-      "logits/chosen": -3.2183594703674316,
+      "logits/chosen": -3.2529296875,
       "logits/rejected": NaN,
-      "logps/chosen": -49.96875,
-      "logps/rejected": -105.32499694824219,
-      "loss": 0.2356,
-      "rewards/accuracies": 0.75390625,
-      "rewards/chosen": -0.3900146484375,
-      "rewards/margins": 2.8427734375,
-      "rewards/rejected": -3.232421875,
+      "logps/chosen": -48.826560974121094,
+      "logps/rejected": -96.4312515258789,
+      "loss": 0.2579,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.3163391053676605,
+      "rewards/margins": 2.467578172683716,
+      "rewards/rejected": -2.783496141433716,
       "step": 100
     },
     {
-      "epoch": 0.3089887640449438,
-      "grad_norm": 0.6719679832458496,
+      "epoch": 0.1544943820224719,
+      "grad_norm": 0.9068161249160767,
       "learning_rate": 4.95663319832678e-07,
-      "logits/chosen": -3.2197265625,
-      "logits/rejected": -2.762890577316284,
-      "logps/chosen": -46.25312423706055,
-      "logps/rejected": -114.41874694824219,
-      "loss": 0.236,
-      "rewards/accuracies": 0.7523437738418579,
-      "rewards/chosen": -0.22943687438964844,
-      "rewards/margins": 3.4632811546325684,
-      "rewards/rejected": -3.6929688453674316,
+      "logits/chosen": -3.236328125,
+      "logits/rejected": -2.843554735183716,
+      "logps/chosen": -45.657814025878906,
+      "logps/rejected": -107.41874694824219,
+      "loss": 0.2278,
+      "rewards/accuracies": 0.753125011920929,
+      "rewards/chosen": -0.1908990889787674,
+      "rewards/margins": 3.155956983566284,
+      "rewards/rejected": -3.34814453125,
       "step": 110
     },
     {
-      "epoch": 0.33707865168539325,
-      "grad_norm": 0.3029685914516449,
+      "epoch": 0.16853932584269662,
+      "grad_norm": 0.5081749558448792,
       "learning_rate": 4.930924800994191e-07,
-      "logits/chosen": -3.2417969703674316,
-      "logits/rejected": -2.752734422683716,
-      "logps/chosen": -45.203125,
-      "logps/rejected": -123.83125305175781,
-      "loss": 0.1955,
-      "rewards/accuracies": 0.77734375,
-      "rewards/chosen": -0.130183607339859,
-      "rewards/margins": 4.001757621765137,
-      "rewards/rejected": -4.1328125,
+      "logits/chosen": -3.252734422683716,
+      "logits/rejected": -2.847460985183716,
+      "logps/chosen": -46.87812423706055,
+      "logps/rejected": -118.3499984741211,
+      "loss": 0.2113,
+      "rewards/accuracies": 0.776562511920929,
+      "rewards/chosen": -0.22010573744773865,
+      "rewards/margins": 3.623046875,
+      "rewards/rejected": -3.8433594703674316,
       "step": 120
     },
     {
-      "epoch": 0.3651685393258427,
-      "grad_norm": 0.5936453938484192,
+      "epoch": 0.18258426966292135,
+      "grad_norm": 0.6637352108955383,
       "learning_rate": 4.899360092892144e-07,
-      "logits/chosen": -3.2191405296325684,
-      "logits/rejected": -2.7181639671325684,
-      "logps/chosen": -46.69843673706055,
-      "logps/rejected": -128.28750610351562,
-      "loss": 0.2039,
-      "rewards/accuracies": 0.7679687738418579,
-      "rewards/chosen": -0.2604345381259918,
-      "rewards/margins": 4.139843940734863,
-      "rewards/rejected": -4.399609565734863,
+      "logits/chosen": -3.2525391578674316,
+      "logits/rejected": -2.763671875,
+      "logps/chosen": -47.01093673706055,
+      "logps/rejected": -124.7437515258789,
+      "loss": 0.2106,
+      "rewards/accuracies": 0.7671874761581421,
+      "rewards/chosen": -0.23588410019874573,
+      "rewards/margins": 3.9537110328674316,
+      "rewards/rejected": -4.188086032867432,
       "step": 130
     },
     {
-      "epoch": 0.39325842696629215,
-      "grad_norm": 0.4349266588687897,
+      "epoch": 0.19662921348314608,
+      "grad_norm": 0.3586008548736572,
       "learning_rate": 4.862015116167195e-07,
-      "logits/chosen": -3.243945360183716,
-      "logits/rejected": NaN,
-      "logps/chosen": -45.373435974121094,
-      "logps/rejected": -133.71249389648438,
-      "loss": 0.1869,
-      "rewards/accuracies": 0.7789062261581421,
-      "rewards/chosen": -0.10051727294921875,
-      "rewards/margins": 4.529296875,
-      "rewards/rejected": -4.631054878234863,
+      "logits/chosen": -3.271484375,
+      "logits/rejected": -2.750781297683716,
+      "logps/chosen": -42.41093826293945,
+      "logps/rejected": -132.1125030517578,
+      "loss": 0.182,
+      "rewards/accuracies": 0.785937488079071,
+      "rewards/chosen": 0.008197021670639515,
+      "rewards/margins": 4.537890434265137,
+      "rewards/rejected": -4.529687404632568,
       "step": 140
     },
     {
-      "epoch": 0.42134831460674155,
-      "grad_norm": 0.32329249382019043,
+      "epoch": 0.21067415730337077,
+      "grad_norm": 0.7338384985923767,
       "learning_rate": 4.81897983813931e-07,
-      "logits/chosen": -3.2386717796325684,
-      "logits/rejected": -2.702929735183716,
-      "logps/chosen": -46.21562576293945,
-      "logps/rejected": -138.9375,
-      "loss": 0.1833,
-      "rewards/accuracies": 0.780468761920929,
-      "rewards/chosen": -0.20498999953269958,
-      "rewards/margins": 4.690625190734863,
-      "rewards/rejected": -4.895898342132568,
+      "logits/chosen": -3.2681641578674316,
+      "logits/rejected": NaN,
+      "logps/chosen": -39.69062423706055,
+      "logps/rejected": -129.3562469482422,
+      "loss": 0.1964,
+      "rewards/accuracies": 0.7593749761581421,
+      "rewards/chosen": 0.10731048882007599,
+      "rewards/margins": 4.583203315734863,
+      "rewards/rejected": -4.477246284484863,
       "step": 150
     },
     {
-      "epoch": 0.449438202247191,
-      "grad_norm": 0.24011513590812683,
+      "epoch": 0.2247191011235955,
+      "grad_norm": 0.7118776440620422,
       "learning_rate": 4.770357934562704e-07,
-      "logits/chosen": -3.281445264816284,
-      "logits/rejected": NaN,
-      "logps/chosen": -41.82500076293945,
-      "logps/rejected": -132.2624969482422,
-      "loss": 0.195,
-      "rewards/accuracies": 0.765625,
-      "rewards/chosen": 0.00346546177752316,
-      "rewards/margins": 4.6083984375,
-      "rewards/rejected": -4.605273246765137,
+      "logits/chosen": -3.279296875,
+      "logits/rejected": -2.6996092796325684,
+      "logps/chosen": -44.157814025878906,
+      "logps/rejected": -138.63125610351562,
+      "loss": 0.1941,
+      "rewards/accuracies": 0.7828124761581421,
+      "rewards/chosen": -0.10118408501148224,
+      "rewards/margins": 4.762890815734863,
+      "rewards/rejected": -4.8642578125,
       "step": 160
     },
     {
-      "epoch": 0.47752808988764045,
-      "grad_norm": 0.5602438449859619,
+      "epoch": 0.23876404494382023,
+      "grad_norm": 0.20937258005142212,
       "learning_rate": 4.716266539861866e-07,
-      "logits/chosen": -3.237499952316284,
-      "logits/rejected": -2.6552734375,
-      "logps/chosen": -47.9921875,
-      "logps/rejected": -144.77499389648438,
-      "loss": 0.1834,
-      "rewards/accuracies": 0.7757812738418579,
-      "rewards/chosen": -0.31496095657348633,
-      "rewards/margins": 4.906640529632568,
-      "rewards/rejected": -5.219336032867432,
+      "logits/chosen": -3.1851563453674316,
+      "logits/rejected": -2.635546922683716,
+      "logps/chosen": -51.939064025878906,
+      "logps/rejected": -140.15625,
+      "loss": 0.1944,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.5132009387016296,
+      "rewards/margins": 4.476171970367432,
+      "rewards/rejected": -4.986914157867432,
       "step": 170
     },
     {
-      "epoch": 0.5056179775280899,
-      "grad_norm": 0.5013967156410217,
+      "epoch": 0.25280898876404495,
+      "grad_norm": 1.1107814311981201,
       "learning_rate": 4.6568359649444796e-07,
-      "logits/chosen": -3.225781202316284,
-      "logits/rejected": -2.6431641578674316,
-      "logps/chosen": -47.4140625,
-      "logps/rejected": -147.6062469482422,
-      "loss": 0.1777,
-      "rewards/accuracies": 0.7835937738418579,
-      "rewards/chosen": -0.26249465346336365,
-      "rewards/margins": 5.070898532867432,
-      "rewards/rejected": -5.333398342132568,
+      "logits/chosen": -3.2699217796325684,
+      "logits/rejected": -2.663281202316284,
+      "logps/chosen": -37.44843673706055,
+      "logps/rejected": -139.5437469482422,
+      "loss": 0.1877,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": 0.19300994277000427,
+      "rewards/margins": 5.172265529632568,
+      "rewards/rejected": -4.980273246765137,
       "step": 180
     },
     {
-      "epoch": 0.5337078651685393,
-      "grad_norm": 0.3109186887741089,
+      "epoch": 0.26685393258426965,
+      "grad_norm": 0.1760079711675644,
       "learning_rate": 4.592209383271023e-07,
-      "logits/chosen": -3.2769532203674316,
-      "logits/rejected": -2.708203077316284,
-      "logps/chosen": -40.32500076293945,
-      "logps/rejected": -146.55624389648438,
-      "loss": 0.1691,
-      "rewards/accuracies": 0.7945312261581421,
-      "rewards/chosen": 0.10864410549402237,
-      "rewards/margins": 5.372851371765137,
-      "rewards/rejected": -5.262499809265137,
+      "logits/chosen": -3.15234375,
+      "logits/rejected": NaN,
+      "logps/chosen": -50.7109375,
+      "logps/rejected": -142.33749389648438,
+      "loss": 0.2011,
+      "rewards/accuracies": 0.753125011920929,
+      "rewards/chosen": -0.4693801999092102,
+      "rewards/margins": 4.672265529632568,
+      "rewards/rejected": -5.140820503234863,
       "step": 190
     },
     {
-      "epoch": 0.5617977528089888,
-      "grad_norm": 0.329673707485199,
+      "epoch": 0.2808988764044944,
+      "grad_norm": 0.3540495038032532,
       "learning_rate": 4.5225424859373684e-07,
-      "logits/chosen": -3.244921922683716,
-      "logits/rejected": -2.625195264816284,
-      "logps/chosen": -46.1875,
-      "logps/rejected": -152.55624389648438,
-      "loss": 0.1722,
-      "rewards/accuracies": 0.784375011920929,
-      "rewards/chosen": -0.20301513373851776,
-      "rewards/margins": 5.3759765625,
-      "rewards/rejected": -5.578711032867432,
+      "logits/chosen": -3.2144532203674316,
+      "logits/rejected": -2.646484375,
+      "logps/chosen": -44.23749923706055,
+      "logps/rejected": -146.5,
+      "loss": 0.1823,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.061879731714725494,
+      "rewards/margins": 5.176171779632568,
+      "rewards/rejected": -5.240038871765137,
       "step": 200
     },
     {
-      "epoch": 0.5898876404494382,
-      "grad_norm": 0.6182069182395935,
+      "epoch": 0.2949438202247191,
+      "grad_norm": 0.4451713263988495,
       "learning_rate": 4.448003106601291e-07,
-      "logits/chosen": -3.2255859375,
-      "logits/rejected": NaN,
-      "logps/chosen": -45.904685974121094,
-      "logps/rejected": -153.14999389648438,
-      "loss": 0.1818,
-      "rewards/accuracies": 0.7789062261581421,
-      "rewards/chosen": -0.2197813093662262,
-      "rewards/margins": 5.441601753234863,
-      "rewards/rejected": -5.661913871765137,
+      "logits/chosen": -3.2412109375,
+      "logits/rejected": -2.6431641578674316,
+      "logps/chosen": -39.69843673706055,
+      "logps/rejected": -141.1062469482422,
+      "loss": 0.2101,
+      "rewards/accuracies": 0.7671874761581421,
+      "rewards/chosen": 0.11627502739429474,
+      "rewards/margins": 5.123827934265137,
+      "rewards/rejected": -5.008593559265137,
       "step": 210
     },
     {
-      "epoch": 0.6179775280898876,
-      "grad_norm": 0.10240374505519867,
+      "epoch": 0.3089887640449438,
+      "grad_norm": 1.5930163860321045,
       "learning_rate": 4.3687708171564917e-07,
-      "logits/chosen": -3.2201170921325684,
-      "logits/rejected": NaN,
-      "logps/chosen": -46.890625,
-      "logps/rejected": -154.24374389648438,
-      "loss": 0.18,
-      "rewards/accuracies": 0.76953125,
-      "rewards/chosen": -0.2859039306640625,
-      "rewards/margins": 5.441992282867432,
-      "rewards/rejected": -5.730078220367432,
+      "logits/chosen": -3.2291016578674316,
+      "logits/rejected": -2.604296922683716,
+      "logps/chosen": -42.740623474121094,
+      "logps/rejected": -145.375,
+      "loss": 0.2042,
+      "rewards/accuracies": 0.7578125,
+      "rewards/chosen": -0.070429228246212,
+      "rewards/margins": 5.188672065734863,
+      "rewards/rejected": -5.260156154632568,
       "step": 220
     },
     {
-      "epoch": 0.6460674157303371,
-      "grad_norm": 0.5484092235565186,
+      "epoch": 0.32303370786516855,
+      "grad_norm": 0.27790066599845886,
       "learning_rate": 4.2850364951281705e-07,
-      "logits/chosen": -3.2359375953674316,
-      "logits/rejected": -2.595507860183716,
-      "logps/chosen": -44.52656173706055,
-      "logps/rejected": -156.5187530517578,
-      "loss": 0.1763,
-      "rewards/accuracies": 0.785937488079071,
-      "rewards/chosen": -0.10360870510339737,
-      "rewards/margins": 5.668359279632568,
-      "rewards/rejected": -5.7724609375,
+      "logits/chosen": -3.2845702171325684,
+      "logits/rejected": -2.6654295921325684,
+      "logps/chosen": -39.85625076293945,
+      "logps/rejected": -149.9250030517578,
+      "loss": 0.1742,
+      "rewards/accuracies": 0.778124988079071,
+      "rewards/chosen": 0.12183837592601776,
+      "rewards/margins": 5.572656154632568,
+      "rewards/rejected": -5.451562404632568,
       "step": 230
     },
     {
-      "epoch": 0.6741573033707865,
-      "grad_norm": 0.33201783895492554,
+      "epoch": 0.33707865168539325,
+      "grad_norm": 0.32155928015708923,
       "learning_rate": 4.1970018638323547e-07,
-      "logits/chosen": -3.2476563453674316,
-      "logits/rejected": -2.6099610328674316,
-      "logps/chosen": -41.240623474121094,
-      "logps/rejected": -152.3249969482422,
-      "loss": 0.1836,
-      "rewards/accuracies": 0.778124988079071,
-      "rewards/chosen": 0.05380706861615181,
-      "rewards/margins": 5.612500190734863,
-      "rewards/rejected": -5.559960842132568,
+      "logits/chosen": -3.26953125,
+      "logits/rejected": -2.6566405296325684,
+      "logps/chosen": -41.8671875,
+      "logps/rejected": -152.64999389648438,
+      "loss": 0.174,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.051790811121463776,
+      "rewards/margins": 5.608593940734863,
+      "rewards/rejected": -5.559179782867432,
       "step": 240
     },
     {
-      "epoch": 0.702247191011236,
-      "grad_norm": 0.3225279450416565,
+      "epoch": 0.351123595505618,
+      "grad_norm": 0.2504253685474396,
       "learning_rate": 4.1048790064067573e-07,
-      "logits/chosen": -3.2152342796325684,
-      "logits/rejected": NaN,
-      "logps/chosen": -42.44062423706055,
-      "logps/rejected": -152.5437469482422,
-      "loss": 0.1874,
-      "rewards/accuracies": 0.7679687738418579,
-      "rewards/chosen": -0.019408416002988815,
-      "rewards/margins": 5.590234279632568,
-      "rewards/rejected": -5.608788967132568,
+      "logits/chosen": -3.254687547683716,
+      "logits/rejected": -2.630078077316284,
+      "logps/chosen": -40.02812576293945,
+      "logps/rejected": -154.9250030517578,
+      "loss": 0.1722,
+      "rewards/accuracies": 0.785937488079071,
+      "rewards/chosen": 0.126708984375,
+      "rewards/margins": 5.812109470367432,
+      "rewards/rejected": -5.681640625,
       "step": 250
     },
     {
-      "epoch": 0.7303370786516854,
-      "grad_norm": 0.5013627409934998,
+      "epoch": 0.3651685393258427,
+      "grad_norm": 1.5456242561340332,
       "learning_rate": 4.0088898548839285e-07,
-      "logits/chosen": -3.1996092796325684,
-      "logits/rejected": -2.558398485183716,
-      "logps/chosen": -46.775001525878906,
-      "logps/rejected": -157.64999389648438,
-      "loss": 0.1809,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -0.24601340293884277,
-      "rewards/margins": 5.5927734375,
-      "rewards/rejected": -5.837695121765137,
+      "logits/chosen": -3.2398438453674316,
+      "logits/rejected": -2.6322264671325684,
+      "logps/chosen": -42.142189025878906,
+      "logps/rejected": -149.5749969482422,
+      "loss": 0.1996,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.08585663139820099,
+      "rewards/margins": 5.431640625,
+      "rewards/rejected": -5.516211032867432,
       "step": 260
     },
     {
-      "epoch": 0.7584269662921348,
-      "grad_norm": 0.2542474567890167,
+      "epoch": 0.3792134831460674,
+      "grad_norm": 0.1563502699136734,
       "learning_rate": 3.9092656555375414e-07,
-      "logits/chosen": -3.184765577316284,
-      "logits/rejected": -2.549999952316284,
-      "logps/chosen": -48.16718673706055,
-      "logps/rejected": -155.3937530517578,
-      "loss": 0.1927,
-      "rewards/accuracies": 0.7593749761581421,
-      "rewards/chosen": -0.3481277525424957,
-      "rewards/margins": 5.426171779632568,
-      "rewards/rejected": -5.775000095367432,
+      "logits/chosen": -3.268359422683716,
+      "logits/rejected": -2.6371092796325684,
+      "logps/chosen": -41.34375,
+      "logps/rejected": -160.625,
+      "loss": 0.159,
+      "rewards/accuracies": 0.796875,
+      "rewards/chosen": 0.07463989406824112,
+      "rewards/margins": 6.014843940734863,
+      "rewards/rejected": -5.940234184265137,
       "step": 270
     },
     {
-      "epoch": 0.7865168539325843,
-      "grad_norm": 1.486909031867981,
+      "epoch": 0.39325842696629215,
+      "grad_norm": 0.414302796125412,
       "learning_rate": 3.806246411789872e-07,
-      "logits/chosen": -3.206835985183716,
-      "logits/rejected": -2.568554639816284,
-      "logps/chosen": -46.279685974121094,
-      "logps/rejected": -161.4499969482422,
-      "loss": 0.1695,
-      "rewards/accuracies": 0.796875,
-      "rewards/chosen": -0.17198029160499573,
-      "rewards/margins": 5.812304496765137,
-      "rewards/rejected": -5.983984470367432,
+      "logits/chosen": -3.282421827316284,
+      "logits/rejected": NaN,
+      "logps/chosen": -39.970314025878906,
+      "logps/rejected": -151.84375,
+      "loss": 0.1905,
+      "rewards/accuracies": 0.7640625238418579,
+      "rewards/chosen": 0.1959686279296875,
+      "rewards/margins": 5.774609565734863,
+      "rewards/rejected": -5.579492092132568,
       "step": 280
     },
     {
-      "epoch": 0.8146067415730337,
-      "grad_norm": 0.24136996269226074,
+      "epoch": 0.40730337078651685,
+      "grad_norm": 0.11994462460279465,
       "learning_rate": 3.700080306022528e-07,
-      "logits/chosen": -3.238085985183716,
-      "logits/rejected": -2.606250047683716,
-      "logps/chosen": -42.40312576293945,
-      "logps/rejected": -155.80624389648438,
-      "loss": 0.1873,
-      "rewards/accuracies": 0.76953125,
-      "rewards/chosen": -0.016921233385801315,
-      "rewards/margins": 5.745703220367432,
-      "rewards/rejected": -5.7607421875,
+      "logits/chosen": -3.199023485183716,
+      "logits/rejected": -2.582226514816284,
+      "logps/chosen": -51.41093826293945,
+      "logps/rejected": -164.3125,
+      "loss": 0.1673,
+      "rewards/accuracies": 0.7890625,
+      "rewards/chosen": -0.46795958280563354,
+      "rewards/margins": 5.684179782867432,
+      "rewards/rejected": -6.149609565734863,
       "step": 290
     },
     {
-      "epoch": 0.8426966292134831,
-      "grad_norm": 0.3395313620567322,
+      "epoch": 0.42134831460674155,
+      "grad_norm": 0.3142966330051422,
       "learning_rate": 3.5910231016833546e-07,
-      "logits/chosen": -3.2119140625,
-      "logits/rejected": NaN,
-      "logps/chosen": -45.810935974121094,
-      "logps/rejected": -161.1687469482422,
-      "loss": 0.174,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -0.18533477187156677,
-      "rewards/margins": 5.834374904632568,
-      "rewards/rejected": -6.019140720367432,
+      "logits/chosen": -3.1810545921325684,
+      "logits/rejected": -2.5703125,
+      "logps/chosen": -50.90625,
+      "logps/rejected": -161.7375030517578,
+      "loss": 0.1775,
+      "rewards/accuracies": 0.7796875238418579,
+      "rewards/chosen": -0.4352920651435852,
+      "rewards/margins": 5.612890720367432,
+      "rewards/rejected": -6.050000190734863,
       "step": 300
     },
     {
-      "epoch": 0.8707865168539326,
-      "grad_norm": 0.14939536154270172,
+      "epoch": 0.4353932584269663,
+      "grad_norm": 0.5336220264434814,
       "learning_rate": 3.4793375271298895e-07,
-      "logits/chosen": -3.1988282203674316,
-      "logits/rejected": -2.597851514816284,
-      "logps/chosen": -44.76874923706055,
-      "logps/rejected": -163.0437469482422,
-      "loss": 0.1656,
-      "rewards/accuracies": 0.7867187261581421,
-      "rewards/chosen": -0.11381302028894424,
-      "rewards/margins": 5.981835842132568,
-      "rewards/rejected": -6.096875190734863,
+      "logits/chosen": -3.252148389816284,
+      "logits/rejected": NaN,
+      "logps/chosen": -41.959373474121094,
+      "logps/rejected": -157.24374389648438,
+      "loss": 0.1855,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": 0.02679443359375,
+      "rewards/margins": 5.8720703125,
+      "rewards/rejected": -5.84375,
       "step": 310
     },
     {
-      "epoch": 0.898876404494382,
-      "grad_norm": 0.6819726228713989,
+      "epoch": 0.449438202247191,
+      "grad_norm": 0.11426942050457001,
       "learning_rate": 3.3652926426937325e-07,
-      "logits/chosen": -3.246875047683716,
-      "logits/rejected": -2.609179735183716,
-      "logps/chosen": -41.875,
-      "logps/rejected": -162.28125,
-      "loss": 0.1824,
-      "rewards/accuracies": 0.7828124761581421,
-      "rewards/chosen": 0.06156463548541069,
-      "rewards/margins": 6.103515625,
-      "rewards/rejected": -6.043359279632568,
+      "logits/chosen": -3.280078172683716,
+      "logits/rejected": -2.6470704078674316,
+      "logps/chosen": -40.0546875,
+      "logps/rejected": -156.4812469482422,
+      "loss": 0.1744,
+      "rewards/accuracies": 0.770312488079071,
+      "rewards/chosen": 0.06229095533490181,
+      "rewards/margins": 5.884179592132568,
+      "rewards/rejected": -5.8212890625,
       "step": 320
     },
     {
-      "epoch": 0.9269662921348315,
-      "grad_norm": 0.10508494079113007,
+      "epoch": 0.46348314606741575,
+      "grad_norm": 0.12852512300014496,
       "learning_rate": 3.249163192490642e-07,
-      "logits/chosen": -3.2544922828674316,
-      "logits/rejected": -2.624804735183716,
-      "logps/chosen": -41.25,
-      "logps/rejected": -159.97500610351562,
-      "loss": 0.1816,
-      "rewards/accuracies": 0.7734375,
-      "rewards/chosen": 0.021668624132871628,
-      "rewards/margins": 5.989648342132568,
-      "rewards/rejected": -5.967968940734863,
+      "logits/chosen": -3.245898485183716,
+      "logits/rejected": -2.5863280296325684,
+      "logps/chosen": -43.52031326293945,
+      "logps/rejected": -163.125,
+      "loss": 0.1654,
+      "rewards/accuracies": 0.785937488079071,
+      "rewards/chosen": -0.09057464450597763,
+      "rewards/margins": 6.042578220367432,
+      "rewards/rejected": -6.135937690734863,
       "step": 330
     },
     {
-      "epoch": 0.9550561797752809,
-      "grad_norm": 0.2005680650472641,
+      "epoch": 0.47752808988764045,
+      "grad_norm": 1.0368801355361938,
       "learning_rate": 3.1312289425378944e-07,
-      "logits/chosen": -3.2759766578674316,
-      "logits/rejected": -2.6624999046325684,
-      "logps/chosen": -39.45781326293945,
-      "logps/rejected": -157.4375,
-      "loss": 0.1757,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": 0.14164963364601135,
-      "rewards/margins": 5.972851753234863,
-      "rewards/rejected": -5.831250190734863,
+      "logits/chosen": -3.237109422683716,
+      "logits/rejected": -2.5757813453674316,
+      "logps/chosen": -45.65156173706055,
+      "logps/rejected": -166.43124389648438,
+      "loss": 0.1851,
+      "rewards/accuracies": 0.7718750238418579,
+      "rewards/chosen": -0.196772962808609,
+      "rewards/margins": 6.102734565734863,
+      "rewards/rejected": -6.302343845367432,
       "step": 340
     },
     {
-      "epoch": 0.9831460674157303,
-      "grad_norm": 0.2797287404537201,
+      "epoch": 0.49157303370786515,
+      "grad_norm": 0.2768089473247528,
       "learning_rate": 3.011774006773449e-07,
-      "logits/chosen": -3.282031297683716,
-      "logits/rejected": NaN,
-      "logps/chosen": -37.98906326293945,
-      "logps/rejected": -157.4375,
-      "loss": 0.1713,
-      "rewards/accuracies": 0.7757812738418579,
-      "rewards/chosen": 0.19436034560203552,
-      "rewards/margins": 6.061718940734863,
-      "rewards/rejected": -5.866406440734863,
+      "logits/chosen": -3.1851563453674316,
+      "logits/rejected": -2.553515672683716,
+      "logps/chosen": -50.12968826293945,
+      "logps/rejected": -172.3125,
+      "loss": 0.1616,
+      "rewards/accuracies": 0.801562488079071,
+      "rewards/chosen": -0.33214110136032104,
+      "rewards/margins": 6.178515434265137,
+      "rewards/rejected": -6.510156154632568,
       "step": 350
     },
     {
-      "epoch": 1.0112359550561798,
-      "grad_norm": 0.1386144906282425,
+      "epoch": 0.5056179775280899,
+      "grad_norm": 0.5607307553291321,
       "learning_rate": 2.8910861626005773e-07,
-      "logits/chosen": -3.2330079078674316,
-      "logits/rejected": -2.6128907203674316,
-      "logps/chosen": -42.69062423706055,
-      "logps/rejected": -162.75625610351562,
-      "loss": 0.1689,
-      "rewards/accuracies": 0.7942708730697632,
-      "rewards/chosen": -0.019468307495117188,
-      "rewards/margins": 6.076367378234863,
-      "rewards/rejected": -6.09765625,
+      "logits/chosen": -3.2269530296325684,
+      "logits/rejected": -2.5611329078674316,
+      "logps/chosen": -45.09375,
+      "logps/rejected": -164.16250610351562,
+      "loss": 0.1745,
+      "rewards/accuracies": 0.7718750238418579,
+      "rewards/chosen": -0.21467895805835724,
+      "rewards/margins": 6.003515720367432,
+      "rewards/rejected": -6.216406345367432,
       "step": 360
     },
     {
-      "epoch": 1.0393258426966292,
-      "grad_norm": 0.31265443563461304,
+      "epoch": 0.5196629213483146,
+      "grad_norm": 0.45973560214042664,
       "learning_rate": 2.7694561576068983e-07,
-      "logits/chosen": -3.234375,
-      "logits/rejected": -2.5785155296325684,
-      "logps/chosen": -41.896873474121094,
-      "logps/rejected": -160.8125,
-      "loss": 0.1713,
-      "rewards/accuracies": 0.7867187261581421,
-      "rewards/chosen": -0.01103897113353014,
-      "rewards/margins": 6.00390625,
-      "rewards/rejected": -6.014843940734863,
+      "logits/chosen": -3.203906297683716,
+      "logits/rejected": -2.5244140625,
+      "logps/chosen": -46.931251525878906,
+      "logps/rejected": -171.2375030517578,
+      "loss": 0.1627,
+      "rewards/accuracies": 0.796875,
+      "rewards/chosen": -0.20502586662769318,
+      "rewards/margins": 6.284570217132568,
+      "rewards/rejected": -6.489062309265137,
       "step": 370
     },
     {
-      "epoch": 1.0674157303370786,
-      "grad_norm": 0.3972986042499542,
+      "epoch": 0.5337078651685393,
+      "grad_norm": 0.33313286304473877,
       "learning_rate": 2.647177009127972e-07,
-      "logits/chosen": -3.2544922828674316,
-      "logits/rejected": -2.5933594703674316,
-      "logps/chosen": -39.5703125,
-      "logps/rejected": -160.85000610351562,
-      "loss": 0.1793,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": 0.09815521538257599,
-      "rewards/margins": 6.125781059265137,
-      "rewards/rejected": -6.029492378234863,
+      "logits/chosen": -3.1996092796325684,
+      "logits/rejected": -2.5083985328674316,
+      "logps/chosen": -46.064064025878906,
+      "logps/rejected": -171.14999389648438,
+      "loss": 0.1645,
+      "rewards/accuracies": 0.7984374761581421,
+      "rewards/chosen": -0.1926528960466385,
+      "rewards/margins": 6.312890529632568,
+      "rewards/rejected": -6.504101753234863,
       "step": 380
     },
     {
-      "epoch": 1.095505617977528,
-      "grad_norm": 0.12721529603004456,
+      "epoch": 0.547752808988764,
+      "grad_norm": 0.3406722843647003,
       "learning_rate": 2.524543298342874e-07,
-      "logits/chosen": -3.262890577316284,
-      "logits/rejected": -2.592578172683716,
-      "logps/chosen": -40.4765625,
-      "logps/rejected": -161.91250610351562,
-      "loss": 0.1743,
-      "rewards/accuracies": 0.77734375,
-      "rewards/chosen": 0.05222644656896591,
-      "rewards/margins": 6.142578125,
-      "rewards/rejected": -6.091406345367432,
+      "logits/chosen": -3.216992139816284,
+      "logits/rejected": -2.5326170921325684,
+      "logps/chosen": -43.662498474121094,
+      "logps/rejected": -168.7937469482422,
+      "loss": 0.1589,
+      "rewards/accuracies": 0.792187511920929,
+      "rewards/chosen": -0.07064209133386612,
+      "rewards/margins": 6.324023246765137,
+      "rewards/rejected": -6.3935546875,
       "step": 390
     },
     {
-      "epoch": 1.1235955056179776,
-      "grad_norm": 0.09501045197248459,
+      "epoch": 0.5617977528089888,
+      "grad_norm": 0.2776348292827606,
       "learning_rate": 2.401850460602329e-07,
-      "logits/chosen": -3.2220702171325684,
-      "logits/rejected": -2.559765577316284,
-      "logps/chosen": -44.423439025878906,
-      "logps/rejected": -167.08749389648438,
-      "loss": 0.1718,
-      "rewards/accuracies": 0.780468761920929,
-      "rewards/chosen": -0.09567908942699432,
-      "rewards/margins": 6.20703125,
-      "rewards/rejected": -6.302734375,
+      "logits/chosen": -3.2457032203674316,
+      "logits/rejected": -2.5503907203674316,
+      "logps/chosen": -41.400001525878906,
+      "logps/rejected": -166.88125610351562,
+      "loss": 0.1747,
+      "rewards/accuracies": 0.7828124761581421,
+      "rewards/chosen": 0.02814788743853569,
+      "rewards/margins": 6.317187309265137,
+      "rewards/rejected": -6.288281440734863,
       "step": 400
     },
     {
-      "epoch": 1.151685393258427,
-      "grad_norm": 0.29580923914909363,
+      "epoch": 0.5758426966292135,
+      "grad_norm": 1.0503939390182495,
       "learning_rate": 2.2793940736990766e-07,
-      "logits/chosen": -3.2007813453674316,
-      "logits/rejected": NaN,
-      "logps/chosen": -51.12968826293945,
-      "logps/rejected": -170.96875,
-      "loss": 0.172,
-      "rewards/accuracies": 0.780468761920929,
-      "rewards/chosen": -0.42823487520217896,
-      "rewards/margins": 6.079297065734863,
-      "rewards/rejected": -6.508593559265137,
+      "logits/chosen": -3.241992235183716,
+      "logits/rejected": -2.546093702316284,
+      "logps/chosen": -40.43437576293945,
+      "logps/rejected": -163.3874969482422,
+      "loss": 0.1881,
+      "rewards/accuracies": 0.7718750238418579,
+      "rewards/chosen": 0.005574035458266735,
+      "rewards/margins": 6.197851657867432,
+      "rewards/rejected": -6.190625190734863,
       "step": 410
     },
     {
-      "epoch": 1.1797752808988764,
-      "grad_norm": 0.2765229046344757,
+      "epoch": 0.5898876404494382,
+      "grad_norm": 1.4323946237564087,
       "learning_rate": 2.1574691457950803e-07,
-      "logits/chosen": -3.1839842796325684,
+      "logits/chosen": -3.208203077316284,
       "logits/rejected": NaN,
-      "logps/chosen": -50.9609375,
-      "logps/rejected": -170.78125,
-      "loss": 0.1704,
-      "rewards/accuracies": 0.7734375,
-      "rewards/chosen": -0.46327972412109375,
-      "rewards/margins": 6.071484565734863,
-      "rewards/rejected": -6.534375190734863,
+      "logps/chosen": -43.95781326293945,
+      "logps/rejected": -171.10000610351562,
+      "loss": 0.1664,
+      "rewards/accuracies": 0.7890625,
+      "rewards/chosen": -0.07436218112707138,
+      "rewards/margins": 6.463086128234863,
+      "rewards/rejected": -6.539453029632568,
       "step": 420
     },
     {
-      "epoch": 1.2078651685393258,
-      "grad_norm": 0.15300235152244568,
+      "epoch": 0.6039325842696629,
+      "grad_norm": 0.678728461265564,
       "learning_rate": 2.036369404721023e-07,
-      "logits/chosen": -3.2113280296325684,
-      "logits/rejected": -2.531054735183716,
-      "logps/chosen": -46.8828125,
-      "logps/rejected": -173.1374969482422,
-      "loss": 0.1599,
-      "rewards/accuracies": 0.79296875,
-      "rewards/chosen": -0.21221771836280823,
-      "rewards/margins": 6.374609470367432,
-      "rewards/rejected": -6.587109565734863,
+      "logits/chosen": -3.1888670921325684,
+      "logits/rejected": NaN,
+      "logps/chosen": -47.20624923706055,
+      "logps/rejected": -169.1062469482422,
+      "loss": 0.166,
+      "rewards/accuracies": 0.778124988079071,
+      "rewards/chosen": -0.290365606546402,
+      "rewards/margins": 6.1806640625,
+      "rewards/rejected": -6.469531059265137,
       "step": 430
     },
     {
-      "epoch": 1.2359550561797752,
-      "grad_norm": 0.6692606806755066,
+      "epoch": 0.6179775280898876,
+      "grad_norm": 0.20119936764240265,
       "learning_rate": 1.9163865903602372e-07,
-      "logits/chosen": -3.253124952316284,
-      "logits/rejected": -2.5308594703674316,
-      "logps/chosen": -42.287498474121094,
-      "logps/rejected": -169.625,
-      "loss": 0.1762,
-      "rewards/accuracies": 0.776562511920929,
-      "rewards/chosen": -0.029553985223174095,
-      "rewards/margins": 6.40625,
-      "rewards/rejected": -6.434765815734863,
+      "logits/chosen": -3.2255859375,
+      "logits/rejected": -2.520703077316284,
+      "logps/chosen": -45.240623474121094,
+      "logps/rejected": -167.2624969482422,
+      "loss": 0.1816,
+      "rewards/accuracies": 0.7671874761581421,
+      "rewards/chosen": -0.21324768662452698,
+      "rewards/margins": 6.177538871765137,
+      "rewards/rejected": -6.390625,
       "step": 440
     },
     {
-      "epoch": 1.2640449438202248,
-      "grad_norm": 0.2745380103588104,
+      "epoch": 0.6320224719101124,
+      "grad_norm": 0.7942313551902771,
       "learning_rate": 1.7978097518217702e-07,
-      "logits/chosen": -3.237499952316284,
-      "logits/rejected": -2.541015625,
-      "logps/chosen": -42.74687576293945,
-      "logps/rejected": -168.78750610351562,
-      "loss": 0.1715,
-      "rewards/accuracies": 0.774218738079071,
-      "rewards/chosen": -0.021244239062070847,
-      "rewards/margins": 6.392187595367432,
-      "rewards/rejected": -6.409375190734863,
+      "logits/chosen": -3.1763672828674316,
+      "logits/rejected": -2.5126953125,
+      "logps/chosen": -49.80937576293945,
+      "logps/rejected": -172.6374969482422,
+      "loss": 0.1685,
+      "rewards/accuracies": 0.796875,
+      "rewards/chosen": -0.3480590879917145,
+      "rewards/margins": 6.218359470367432,
+      "rewards/rejected": -6.568554878234863,
       "step": 450
     },
     {
-      "epoch": 1.2921348314606742,
-      "grad_norm": 0.3858237564563751,
+      "epoch": 0.6460674157303371,
+      "grad_norm": 0.9307264685630798,
       "learning_rate": 1.6809245510957666e-07,
-      "logits/chosen": -3.235546827316284,
-      "logits/rejected": NaN,
-      "logps/chosen": -44.904685974121094,
-      "logps/rejected": -172.96249389648438,
-      "loss": 0.1663,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -0.10267486423254013,
-      "rewards/margins": 6.484375,
-      "rewards/rejected": -6.587109565734863,
+      "logits/chosen": -3.2232422828674316,
+      "logits/rejected": -2.4925780296325684,
+      "logps/chosen": -44.875,
+      "logps/rejected": -171.28750610351562,
+      "loss": 0.1721,
+      "rewards/accuracies": 0.7796875238418579,
+      "rewards/chosen": -0.14088821411132812,
+      "rewards/margins": 6.381640434265137,
+      "rewards/rejected": -6.520312309265137,
       "step": 460
     },
     {
-      "epoch": 1.3202247191011236,
-      "grad_norm": 0.25496408343315125,
+      "epoch": 0.6601123595505618,
+      "grad_norm": 1.6537760496139526,
       "learning_rate": 1.5660125748687093e-07,
-      "logits/chosen": -3.224414110183716,
-      "logits/rejected": NaN,
-      "logps/chosen": -46.64374923706055,
-      "logps/rejected": -171.1374969482422,
-      "loss": 0.1703,
-      "rewards/accuracies": 0.780468761920929,
-      "rewards/chosen": -0.2242431640625,
-      "rewards/margins": 6.318749904632568,
-      "rewards/rejected": -6.542187690734863,
+      "logits/chosen": -3.2333984375,
+      "logits/rejected": -2.518749952316284,
+      "logps/chosen": -43.334373474121094,
+      "logps/rejected": -167.8125,
+      "loss": 0.1794,
+      "rewards/accuracies": 0.770312488079071,
+      "rewards/chosen": -0.06363830715417862,
+      "rewards/margins": 6.300976753234863,
+      "rewards/rejected": -6.363671779632568,
       "step": 470
     },
     {
-      "epoch": 1.348314606741573,
-      "grad_norm": 0.3984578251838684,
+      "epoch": 0.6741573033707865,
+      "grad_norm": 0.6378122568130493,
       "learning_rate": 1.4533506561564305e-07,
-      "logits/chosen": -3.224609375,
-      "logits/rejected": -2.529101610183716,
-      "logps/chosen": -47.29375076293945,
-      "logps/rejected": -173.3625030517578,
-      "loss": 0.1669,
-      "rewards/accuracies": 0.780468761920929,
-      "rewards/chosen": -0.2605232298374176,
-      "rewards/margins": 6.368359565734863,
-      "rewards/rejected": -6.627734184265137,
+      "logits/chosen": -3.2416014671325684,
+      "logits/rejected": -2.546093702316284,
+      "logps/chosen": -39.32500076293945,
+      "logps/rejected": -166.10000610351562,
+      "loss": 0.1712,
+      "rewards/accuracies": 0.7890625,
+      "rewards/chosen": 0.16349944472312927,
+      "rewards/margins": 6.383008003234863,
+      "rewards/rejected": -6.220312595367432,
       "step": 480
     },
     {
-      "epoch": 1.3764044943820224,
-      "grad_norm": 0.39370596408843994,
+      "epoch": 0.6882022471910112,
+      "grad_norm": 0.21734459698200226,
       "learning_rate": 1.343210207389125e-07,
-      "logits/chosen": -3.203320264816284,
-      "logits/rejected": -2.5302734375,
-      "logps/chosen": -48.0859375,
-      "logps/rejected": -172.93124389648438,
-      "loss": 0.1747,
-      "rewards/accuracies": 0.7757812738418579,
-      "rewards/chosen": -0.27991026639938354,
-      "rewards/margins": 6.329297065734863,
-      "rewards/rejected": -6.610156059265137,
+      "logits/chosen": -3.216015577316284,
+      "logits/rejected": NaN,
+      "logps/chosen": -39.485939025878906,
+      "logps/rejected": -162.99374389648438,
+      "loss": 0.1864,
+      "rewards/accuracies": 0.7640625238418579,
+      "rewards/chosen": 0.08720092475414276,
+      "rewards/margins": 6.254492282867432,
+      "rewards/rejected": -6.165625095367432,
       "step": 490
     },
     {
-      "epoch": 1.404494382022472,
-      "grad_norm": 0.2013687640428543,
+      "epoch": 0.702247191011236,
+      "grad_norm": 0.3136639893054962,
       "learning_rate": 1.2358565665550387e-07,
-      "logits/chosen": -3.244335889816284,
+      "logits/chosen": -3.216015577316284,
       "logits/rejected": NaN,
-      "logps/chosen": -41.96875,
-      "logps/rejected": -176.03750610351562,
-      "loss": 0.1557,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": 0.01244263630360365,
-      "rewards/margins": 6.784765720367432,
-      "rewards/rejected": -6.769921779632568,
+      "logps/chosen": -42.5078125,
+      "logps/rejected": -166.36874389648438,
+      "loss": 0.1841,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.017351532354950905,
+      "rewards/margins": 6.283398628234863,
+      "rewards/rejected": -6.266992092132568,
       "step": 500
     },
     {
-      "epoch": 1.4325842696629214,
-      "grad_norm": 0.24023930728435516,
+      "epoch": 0.7162921348314607,
+      "grad_norm": 0.07860163599252701,
       "learning_rate": 1.1315483579780094e-07,
-      "logits/chosen": -3.2484374046325684,
-      "logits/rejected": -2.5347657203674316,
-      "logps/chosen": -42.046875,
-      "logps/rejected": -175.8312530517578,
-      "loss": 0.1643,
-      "rewards/accuracies": 0.788281261920929,
-      "rewards/chosen": 0.040112875401973724,
-      "rewards/margins": 6.779296875,
-      "rewards/rejected": -6.739453315734863,
+      "logits/chosen": -3.210742235183716,
+      "logits/rejected": -2.499218702316284,
+      "logps/chosen": -41.615623474121094,
+      "logps/rejected": -166.30624389648438,
+      "loss": 0.1831,
+      "rewards/accuracies": 0.7828124761581421,
+      "rewards/chosen": -0.01008453406393528,
+      "rewards/margins": 6.287304878234863,
+      "rewards/rejected": -6.297070503234863,
       "step": 510
     },
     {
-      "epoch": 1.4606741573033708,
-      "grad_norm": 0.5066425800323486,
+      "epoch": 0.7303370786516854,
+      "grad_norm": 0.173310786485672,
       "learning_rate": 1.0305368692688174e-07,
-      "logits/chosen": -3.240429639816284,
-      "logits/rejected": -2.522656202316284,
-      "logps/chosen": -43.00468826293945,
-      "logps/rejected": -173.8625030517578,
-      "loss": 0.1791,
-      "rewards/accuracies": 0.778124988079071,
-      "rewards/chosen": -0.03853149339556694,
-      "rewards/margins": 6.608788967132568,
-      "rewards/rejected": -6.646874904632568,
+      "logits/chosen": -3.195507764816284,
+      "logits/rejected": -2.504101514816284,
+      "logps/chosen": -45.17656326293945,
+      "logps/rejected": -168.6062469482422,
+      "loss": 0.1724,
+      "rewards/accuracies": 0.7828124761581421,
+      "rewards/chosen": -0.14428405463695526,
+      "rewards/margins": 6.204297065734863,
+      "rewards/rejected": -6.347070217132568,
       "step": 520
     },
     {
-      "epoch": 1.4887640449438202,
-      "grad_norm": 0.10324753075838089,
+      "epoch": 0.7443820224719101,
+      "grad_norm": 0.3090410828590393,
       "learning_rate": 9.330654459513266e-08,
-      "logits/chosen": -3.233203172683716,
-      "logits/rejected": -2.5269532203674316,
-      "logps/chosen": -41.953125,
-      "logps/rejected": -172.06875610351562,
-      "loss": 0.1683,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.02672729454934597,
-      "rewards/margins": 6.571875095367432,
-      "rewards/rejected": -6.598437309265137,
+      "logits/chosen": -3.169726610183716,
+      "logits/rejected": -2.4736328125,
+      "logps/chosen": -46.984375,
+      "logps/rejected": -162.36874389648438,
+      "loss": 0.2013,
+      "rewards/accuracies": 0.7515624761581421,
+      "rewards/chosen": -0.3089355528354645,
+      "rewards/margins": 5.836718559265137,
+      "rewards/rejected": -6.145312309265137,
       "step": 530
     },
     {
-      "epoch": 1.5168539325842696,
-      "grad_norm": 0.06884948909282684,
+      "epoch": 0.7584269662921348,
+      "grad_norm": 0.249381884932518,
       "learning_rate": 8.393689052217964e-08,
-      "logits/chosen": -3.2310547828674316,
-      "logits/rejected": -2.5240235328674316,
-      "logps/chosen": -41.89531326293945,
-      "logps/rejected": -170.4499969482422,
-      "loss": 0.1774,
+      "logits/chosen": -3.1357421875,
+      "logits/rejected": -2.484179735183716,
+      "logps/chosen": -49.49687576293945,
+      "logps/rejected": -167.35000610351562,
+      "loss": 0.1759,
       "rewards/accuracies": 0.770312488079071,
-      "rewards/chosen": -0.03569946438074112,
-      "rewards/margins": 6.483984470367432,
-      "rewards/rejected": -6.521093845367432,
+      "rewards/chosen": -0.393869012594223,
+      "rewards/margins": 5.953320503234863,
+      "rewards/rejected": -6.347460746765137,
       "step": 540
     },
     {
-      "epoch": 1.5449438202247192,
-      "grad_norm": 0.2964838445186615,
+      "epoch": 0.7724719101123596,
+      "grad_norm": 0.2429146021604538,
       "learning_rate": 7.49672970253691e-08,
-      "logits/chosen": -3.2562499046325684,
-      "logits/rejected": -2.5296874046325684,
-      "logps/chosen": -40.365623474121094,
-      "logps/rejected": -173.3625030517578,
-      "loss": 0.1644,
-      "rewards/accuracies": 0.78515625,
-      "rewards/chosen": 0.06430435180664062,
-      "rewards/margins": 6.696484565734863,
-      "rewards/rejected": -6.630078315734863,
+      "logits/chosen": -3.1318359375,
+      "logits/rejected": -2.4886717796325684,
+      "logps/chosen": -52.857810974121094,
+      "logps/rejected": -171.19375610351562,
+      "loss": 0.1722,
+      "rewards/accuracies": 0.7953125238418579,
+      "rewards/chosen": -0.4801391661167145,
+      "rewards/margins": 5.990429878234863,
+      "rewards/rejected": -6.468359470367432,
       "step": 550
     },
     {
-      "epoch": 1.5730337078651684,
-      "grad_norm": 0.16073837876319885,
+      "epoch": 0.7865168539325843,
+      "grad_norm": 0.5875958204269409,
       "learning_rate": 6.641937264107867e-08,
-      "logits/chosen": -3.2535157203674316,
-      "logits/rejected": -2.5425782203674316,
-      "logps/chosen": -39.43281173706055,
-      "logps/rejected": -167.91250610351562,
-      "loss": 0.1788,
-      "rewards/accuracies": 0.7671874761581421,
-      "rewards/chosen": 0.08403320610523224,
-      "rewards/margins": 6.482421875,
-      "rewards/rejected": -6.396874904632568,
+      "logits/chosen": -3.171093702316284,
+      "logits/rejected": -2.4808592796325684,
+      "logps/chosen": -47.681251525878906,
+      "logps/rejected": -174.71875,
+      "loss": 0.1605,
+      "rewards/accuracies": 0.7984374761581421,
+      "rewards/chosen": -0.26392096281051636,
+      "rewards/margins": 6.382421970367432,
+      "rewards/rejected": -6.64453125,
       "step": 560
     },
     {
-      "epoch": 1.601123595505618,
-      "grad_norm": 0.27828249335289,
+      "epoch": 0.800561797752809,
+      "grad_norm": 0.37553030252456665,
       "learning_rate": 5.831371006785962e-08,
-      "logits/chosen": -3.2699217796325684,
-      "logits/rejected": -2.5228514671325684,
-      "logps/chosen": -39.87187576293945,
-      "logps/rejected": -171.1437530517578,
-      "loss": 0.175,
-      "rewards/accuracies": 0.772656261920929,
-      "rewards/chosen": 0.07085514068603516,
-      "rewards/margins": 6.617578029632568,
-      "rewards/rejected": -6.548047065734863,
+      "logits/chosen": -3.1826171875,
+      "logits/rejected": -2.4847655296325684,
+      "logps/chosen": -48.498435974121094,
+      "logps/rejected": -169.1125030517578,
+      "loss": 0.1773,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.3075141906738281,
+      "rewards/margins": 6.108984470367432,
+      "rewards/rejected": -6.417578220367432,
       "step": 570
     },
     {
-      "epoch": 1.6292134831460674,
-      "grad_norm": 0.386394202709198,
+      "epoch": 0.8146067415730337,
+      "grad_norm": 0.31017985939979553,
       "learning_rate": 5.066983655682325e-08,
-      "logits/chosen": -3.2455077171325684,
-      "logits/rejected": -2.546093702316284,
-      "logps/chosen": -40.72968673706055,
-      "logps/rejected": -168.77499389648438,
-      "loss": 0.1783,
-      "rewards/accuracies": 0.7679687738418579,
-      "rewards/chosen": 0.032080840319395065,
-      "rewards/margins": 6.479296684265137,
-      "rewards/rejected": -6.448828220367432,
+      "logits/chosen": -3.141796827316284,
+      "logits/rejected": -2.505859375,
+      "logps/chosen": -50.5703125,
+      "logps/rejected": -166.83749389648438,
+      "loss": 0.1899,
+      "rewards/accuracies": 0.7671874761581421,
+      "rewards/chosen": -0.4396209716796875,
+      "rewards/margins": 5.888671875,
+      "rewards/rejected": -6.325585842132568,
       "step": 580
     },
     {
-      "epoch": 1.6573033707865168,
-      "grad_norm": 0.3948213756084442,
+      "epoch": 0.8286516853932584,
+      "grad_norm": 0.8791071772575378,
       "learning_rate": 4.3506166868781755e-08,
-      "logits/chosen": -3.2671875953674316,
-      "logits/rejected": -2.521484375,
-      "logps/chosen": -41.20624923706055,
-      "logps/rejected": -172.6875,
-      "loss": 0.1759,
-      "rewards/accuracies": 0.76953125,
-      "rewards/chosen": 0.012204742059111595,
-      "rewards/margins": 6.639843940734863,
-      "rewards/rejected": -6.629296779632568,
+      "logits/chosen": -3.1845703125,
+      "logits/rejected": -2.4937500953674316,
+      "logps/chosen": -48.103126525878906,
+      "logps/rejected": -170.8625030517578,
+      "loss": 0.1698,
+      "rewards/accuracies": 0.785937488079071,
+      "rewards/chosen": -0.27438658475875854,
+      "rewards/margins": 6.2060546875,
+      "rewards/rejected": -6.482421875,
       "step": 590
     },
     {
-      "epoch": 1.6853932584269664,
-      "grad_norm": 0.4554503560066223,
+      "epoch": 0.8426966292134831,
+      "grad_norm": 0.6104283928871155,
       "learning_rate": 3.683995891147695e-08,
-      "logits/chosen": -3.246289014816284,
+      "logits/chosen": -3.176953077316284,
       "logits/rejected": NaN,
-      "logps/chosen": -42.18281173706055,
-      "logps/rejected": -173.86874389648438,
-      "loss": 0.1625,
+      "logps/chosen": -45.896873474121094,
+      "logps/rejected": -168.60000610351562,
+      "loss": 0.1721,
       "rewards/accuracies": 0.784375011920929,
-      "rewards/chosen": 0.0017807006370276213,
-      "rewards/margins": 6.689453125,
-      "rewards/rejected": -6.688672065734863,
+      "rewards/chosen": -0.21481475234031677,
+      "rewards/margins": 6.201952934265137,
+      "rewards/rejected": -6.416015625,
       "step": 600
     },
     {
-      "epoch": 1.7134831460674156,
-      "grad_norm": 0.6775197386741638,
+      "epoch": 0.8567415730337079,
+      "grad_norm": 0.31073251366615295,
       "learning_rate": 3.0687272163768986e-08,
-      "logits/chosen": -3.259960889816284,
-      "logits/rejected": NaN,
-      "logps/chosen": -41.376564025878906,
-      "logps/rejected": -174.4499969482422,
-      "loss": 0.1619,
-      "rewards/accuracies": 0.788281261920929,
-      "rewards/chosen": 0.07793807983398438,
-      "rewards/margins": 6.741796970367432,
-      "rewards/rejected": -6.6640625,
+      "logits/chosen": -3.1650390625,
+      "logits/rejected": -2.513671875,
+      "logps/chosen": -46.66093826293945,
+      "logps/rejected": -170.8625030517578,
+      "loss": 0.1649,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.24439087510108948,
+      "rewards/margins": 6.258593559265137,
+      "rewards/rejected": -6.502148628234863,
       "step": 610
     },
     {
-      "epoch": 1.7415730337078652,
-      "grad_norm": 0.11740182340145111,
+      "epoch": 0.8707865168539326,
+      "grad_norm": 0.2421959638595581,
       "learning_rate": 2.5062928986944676e-08,
-      "logits/chosen": -3.2728514671325684,
-      "logits/rejected": NaN,
-      "logps/chosen": -40.89374923706055,
-      "logps/rejected": -175.0625,
-      "loss": 0.1611,
-      "rewards/accuracies": 0.7835937738418579,
-      "rewards/chosen": 0.07378844916820526,
-      "rewards/margins": 6.797656059265137,
-      "rewards/rejected": -6.723828315734863,
+      "logits/chosen": -3.170117139816284,
+      "logits/rejected": -2.5238280296325684,
+      "logps/chosen": -46.318748474121094,
+      "logps/rejected": -171.50625610351562,
+      "loss": 0.1595,
+      "rewards/accuracies": 0.7953125238418579,
+      "rewards/chosen": -0.15370789170265198,
+      "rewards/margins": 6.348242282867432,
+      "rewards/rejected": -6.501953125,
       "step": 620
     },
     {
-      "epoch": 1.7696629213483146,
-      "grad_norm": 0.23929810523986816,
+      "epoch": 0.8848314606741573,
+      "grad_norm": 0.292059987783432,
       "learning_rate": 1.9980478916351296e-08,
-      "logits/chosen": -3.2544922828674316,
-      "logits/rejected": -2.5478515625,
-      "logps/chosen": -42.296875,
-      "logps/rejected": -174.1374969482422,
-      "loss": 0.1689,
-      "rewards/accuracies": 0.785937488079071,
-      "rewards/chosen": -0.01357345562428236,
-      "rewards/margins": 6.648046970367432,
-      "rewards/rejected": -6.661328315734863,
+      "logits/chosen": -3.185742139816284,
+      "logits/rejected": -2.491992235183716,
+      "logps/chosen": -46.803123474121094,
+      "logps/rejected": -171.0437469482422,
+      "loss": 0.1893,
+      "rewards/accuracies": 0.778124988079071,
+      "rewards/chosen": -0.1725509613752365,
+      "rewards/margins": 6.320703029632568,
+      "rewards/rejected": -6.491796970367432,
       "step": 630
     },
     {
-      "epoch": 1.797752808988764,
-      "grad_norm": 0.06252361834049225,
+      "epoch": 0.898876404494382,
+      "grad_norm": 0.6732786297798157,
       "learning_rate": 1.5452166019378987e-08,
-      "logits/chosen": -3.2582030296325684,
-      "logits/rejected": -2.5250000953674316,
-      "logps/chosen": -39.837501525878906,
-      "logps/rejected": -173.9875030517578,
-      "loss": 0.1693,
-      "rewards/accuracies": 0.776562511920929,
-      "rewards/chosen": 0.06287650763988495,
-      "rewards/margins": 6.756249904632568,
-      "rewards/rejected": -6.696875095367432,
+      "logits/chosen": -3.2027344703674316,
+      "logits/rejected": -2.5123047828674316,
+      "logps/chosen": -46.25,
+      "logps/rejected": -172.4250030517578,
+      "loss": 0.1712,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -0.16961669921875,
+      "rewards/margins": 6.3662109375,
+      "rewards/rejected": -6.537890434265137,
       "step": 640
     },
     {
-      "epoch": 1.8258426966292136,
-      "grad_norm": 0.6466670632362366,
+      "epoch": 0.9129213483146067,
+      "grad_norm": 0.25237613916397095,
       "learning_rate": 1.1488899398429896e-08,
-      "logits/chosen": -3.2587890625,
-      "logits/rejected": -2.5443358421325684,
-      "logps/chosen": -41.12812423706055,
-      "logps/rejected": -175.3125,
-      "loss": 0.1643,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": 0.057323455810546875,
-      "rewards/margins": 6.766015529632568,
-      "rewards/rejected": -6.710156440734863,
+      "logits/chosen": -3.2007813453674316,
+      "logits/rejected": -2.5160155296325684,
+      "logps/chosen": -43.58906173706055,
+      "logps/rejected": -170.09375,
+      "loss": 0.1787,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.10042724758386612,
+      "rewards/margins": 6.383008003234863,
+      "rewards/rejected": -6.482421875,
       "step": 650
     },
     {
-      "epoch": 1.8539325842696628,
-      "grad_norm": 0.364284485578537,
+      "epoch": 0.9269662921348315,
+      "grad_norm": 0.09695342183113098,
       "learning_rate": 8.100226909935059e-09,
-      "logits/chosen": -3.2474608421325684,
-      "logits/rejected": -2.5396485328674316,
-      "logps/chosen": -43.040626525878906,
-      "logps/rejected": -175.02499389648438,
-      "loss": 0.1656,
-      "rewards/accuracies": 0.7906249761581421,
-      "rewards/chosen": -0.026231002062559128,
-      "rewards/margins": 6.670312404632568,
-      "rewards/rejected": -6.696093559265137,
+      "logits/chosen": -3.2056641578674316,
+      "logits/rejected": -2.5228514671325684,
+      "logps/chosen": -43.27656173706055,
+      "logps/rejected": -166.16250610351562,
+      "loss": 0.1876,
+      "rewards/accuracies": 0.7718750238418579,
+      "rewards/chosen": -0.07465209811925888,
+      "rewards/margins": 6.197656154632568,
+      "rewards/rejected": -6.272265434265137,
       "step": 660
     },
     {
-      "epoch": 1.8820224719101124,
-      "grad_norm": 0.1942785233259201,
+      "epoch": 0.9410112359550562,
+      "grad_norm": 0.15045014023780823,
       "learning_rate": 5.2943121627319346e-09,
-      "logits/chosen": -3.252734422683716,
-      "logits/rejected": -2.544140577316284,
-      "logps/chosen": -41.951560974121094,
-      "logps/rejected": -170.24374389648438,
-      "loss": 0.1726,
+      "logits/chosen": -3.207812547683716,
+      "logits/rejected": -2.5152344703674316,
+      "logps/chosen": -43.06562423706055,
+      "logps/rejected": -166.84375,
+      "loss": 0.1818,
       "rewards/accuracies": 0.770312488079071,
-      "rewards/chosen": -0.00887908972799778,
-      "rewards/margins": 6.493359565734863,
-      "rewards/rejected": -6.500390529632568,
+      "rewards/chosen": -0.07712707668542862,
+      "rewards/margins": 6.268164157867432,
+      "rewards/rejected": -6.342382907867432,
       "step": 670
     },
     {
-      "epoch": 1.9101123595505618,
-      "grad_norm": 0.38263440132141113,
+      "epoch": 0.9550561797752809,
+      "grad_norm": 0.31066492199897766,
       "learning_rate": 3.077914851215585e-09,
-      "logits/chosen": -3.257031202316284,
-      "logits/rejected": -2.5390625,
-      "logps/chosen": -41.865623474121094,
-      "logps/rejected": -176.53750610351562,
-      "loss": 0.1582,
-      "rewards/accuracies": 0.796093761920929,
-      "rewards/chosen": 0.024802017956972122,
-      "rewards/margins": 6.796093940734863,
-      "rewards/rejected": -6.771484375,
+      "logits/chosen": -3.2230467796325684,
+      "logits/rejected": -2.5365233421325684,
+      "logps/chosen": -44.46562576293945,
+      "logps/rejected": -170.8625030517578,
+      "loss": 0.1733,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.07046356052160263,
+      "rewards/margins": 6.385156154632568,
+      "rewards/rejected": -6.457812309265137,
       "step": 680
     },
     {
-      "epoch": 1.9382022471910112,
-      "grad_norm": 0.4378679096698761,
+      "epoch": 0.9691011235955056,
+      "grad_norm": 0.11322323232889175,
       "learning_rate": 1.4563744706429514e-09,
-      "logits/chosen": -3.2578125,
-      "logits/rejected": -2.5419921875,
-      "logps/chosen": -40.009376525878906,
-      "logps/rejected": -171.91250610351562,
-      "loss": 0.1703,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.07464599609375,
-      "rewards/margins": 6.642968654632568,
-      "rewards/rejected": -6.567968845367432,
+      "logits/chosen": -3.216015577316284,
+      "logits/rejected": -2.50390625,
+      "logps/chosen": -41.93281173706055,
+      "logps/rejected": -168.16250610351562,
+      "loss": 0.1693,
+      "rewards/accuracies": 0.778124988079071,
+      "rewards/chosen": -0.01323547400534153,
+      "rewards/margins": 6.383593559265137,
+      "rewards/rejected": -6.3974609375,
       "step": 690
     },
     {
-      "epoch": 1.9662921348314608,
-      "grad_norm": 0.19188769161701202,
+      "epoch": 0.9831460674157303,
+      "grad_norm": 0.28659555315971375,
       "learning_rate": 4.3359745382104405e-10,
-      "logits/chosen": -3.271679639816284,
-      "logits/rejected": -2.5396485328674316,
-      "logps/chosen": -41.52031326293945,
-      "logps/rejected": -175.375,
-      "loss": 0.1601,
-      "rewards/accuracies": 0.7906249761581421,
-      "rewards/chosen": 0.01084976177662611,
-      "rewards/margins": 6.757421970367432,
-      "rewards/rejected": -6.750390529632568,
+      "logits/chosen": -3.21484375,
+      "logits/rejected": NaN,
+      "logps/chosen": -42.96406173706055,
+      "logps/rejected": -169.0,
+      "loss": 0.1708,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.045589447021484375,
+      "rewards/margins": 6.397265434265137,
+      "rewards/rejected": -6.444140434265137,
       "step": 700
     },
     {
-      "epoch": 1.99438202247191,
-      "grad_norm": 0.2792583107948303,
+      "epoch": 0.9971910112359551,
+      "grad_norm": 0.5906934142112732,
       "learning_rate": 1.2047760167999133e-11,
-      "logits/chosen": -3.2544922828674316,
-      "logits/rejected": -2.5517578125,
-      "logps/chosen": -42.29999923706055,
-      "logps/rejected": -175.3625030517578,
-      "loss": 0.1594,
-      "rewards/accuracies": 0.7867187261581421,
-      "rewards/chosen": -0.014897918328642845,
-      "rewards/margins": 6.701171875,
-      "rewards/rejected": -6.717187404632568,
+      "logits/chosen": -3.2037110328674316,
+      "logits/rejected": -2.529101610183716,
+      "logps/chosen": -44.228126525878906,
+      "logps/rejected": -170.83749389648438,
+      "loss": 0.1657,
+      "rewards/accuracies": 0.7984374761581421,
+      "rewards/chosen": -0.09162139892578125,
+      "rewards/margins": 6.3896484375,
+      "rewards/rejected": -6.482421875,
       "step": 710
     },
     {
-      "epoch": 2.0,
+      "epoch": 1.0,
       "step": 712,
       "total_flos": 0.0,
-      "train_loss": 0.22515048312672067,
-      "train_runtime": 2958.2669,
-      "train_samples_per_second": 30.802,
-      "train_steps_per_second": 0.241
+      "train_loss": 0.23233672156092827,
+      "train_runtime": 2773.067,
+      "train_samples_per_second": 16.43,
+      "train_steps_per_second": 0.257
     }
   ],
   "logging_steps": 10,
   "max_steps": 712,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 2,
+  "num_train_epochs": 1,
   "save_steps": 500,
   "stateful_callbacks": {
     "TrainerControl": {

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3e47b8bd96f5ed212913b9d45c5a069e52c690acee03a5d1528a81d6eb90ec43
+oid sha256:e31315ddeb7aa7465f2bd16d20b45c43e929c9c6b6b9f79d4646ff479ab0b680
 size 7800