Training in progress, epoch 2, checkpoint

last-checkpoint/global_step2496/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03d4407874468a31609f6ecbbe75bf8700a11c1b5018c0ef16daafbd696656aa
+size 30462473157

last-checkpoint/global_step2496/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ccbc947add30ccdbea986c88513b17ecf225fdef4e5a4052ab5ccd4eb0377c8
+size 30462473157

last-checkpoint/global_step2496/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a307b23c89338775b5aade32ea4fbde1f98601d778b1dedbd65a5087360d9f22
+size 168021

last-checkpoint/global_step2496/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:672f5f5347aba43a2f3bd4f1def683a3231d360da3149605ba803dc97cb2d03e
+size 168021

last-checkpoint/latest

@@ -1 +1 @@
-global_step1248
+global_step2496

last-checkpoint/model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:02a87a068580e17b63efa87796e3cc03c3c80312c9971d47fd2fd4f6d5582a43
+oid sha256:9fc827e0e29fe68064287084a789c280ee07071edfaf802494bfd45a4328d9b4
 size 4877660776

last-checkpoint/model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5c475f356780c768e46eacf970415ab7cc46714ef3732edce1110907dd8f21a3
+oid sha256:17eff878abe01ff745e70abc42b2798b23c5cca5a165c37c10a14930bca0d0c6
 size 4932751008

last-checkpoint/model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a82ea2f8d60eb99e2c2e7b52258f8ad9af5e81e7251a460b3067da29fd499b58
+oid sha256:614d95265342337ab7854c0fec6ee9677c78a2d9c93f081a8c779cc5a910acd8
 size 4330865200

last-checkpoint/model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1f1f53ced200b2511a937c79d9d4878ea6c7d89792cc4144436543f708ee49f1
+oid sha256:5a018f26b180a313b65824cdc68761f2b6f1700c2d360dfc69b6300fcb5e461b
 size 1089994880

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b580656286e8a6f334aced7bdb46499a54f3bb95644a0167405da037afbd894d
+oid sha256:9698021f2d84167912e7be6ba48d3d2b8d6b20894f23319f36df078c03b33a64
 size 14768

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a763d1d109f11374f3725ac97283433a5c2264a51fd11d55a5af0441e79bbe2c
+oid sha256:90a140d1d010220b1679bf6e519f8d3d518cb57331e0e7fb30008dc00e427811
 size 14768

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f509e07aeb2d18a9542d77802086a220c855eaadfa7372ba3c450b3c079e1739
+oid sha256:76861627b7d29ad5e6036c31e8897f76435dd322d24f5d6f9e3f8afd8fab8ced
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.0,
+  "epoch": 2.0,
   "eval_steps": 500,
-  "global_step": 1248,
+  "global_step": 2496,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -1898,6 +1898,1897 @@
       "eval_samples_per_second": 7.837,
       "eval_steps_per_second": 0.985,
       "step": 1248
+    },
+    {
+      "epoch": 1.001602564102564,
+      "grad_norm": 1.5097157370383436e-06,
+      "learning_rate": 3.7013950727218754e-07,
+      "logits/chosen": 0.90234375,
+      "logits/rejected": 1.2890625,
+      "logps/chosen": -149.0,
+      "logps/rejected": -280.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.4375,
+      "rewards/margins": 19.375,
+      "rewards/rejected": -15.875,
+      "step": 1250
+    },
+    {
+      "epoch": 1.0096153846153846,
+      "grad_norm": 6.567245720525397e-06,
+      "learning_rate": 3.6865538735529826e-07,
+      "logits/chosen": 0.8359375,
+      "logits/rejected": 1.046875,
+      "logps/chosen": -135.0,
+      "logps/rejected": -284.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.640625,
+      "rewards/margins": 18.5,
+      "rewards/rejected": -14.875,
+      "step": 1260
+    },
+    {
+      "epoch": 1.017628205128205,
+      "grad_norm": 2.7660339460230295e-07,
+      "learning_rate": 3.6717126743840897e-07,
+      "logits/chosen": 0.75390625,
+      "logits/rejected": 1.203125,
+      "logps/chosen": -176.0,
+      "logps/rejected": -296.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.71875,
+      "rewards/margins": 19.25,
+      "rewards/rejected": -15.5,
+      "step": 1270
+    },
+    {
+      "epoch": 1.0256410256410255,
+      "grad_norm": 0.00038016083445366024,
+      "learning_rate": 3.6568714752151974e-07,
+      "logits/chosen": 0.90625,
+      "logits/rejected": 1.203125,
+      "logps/chosen": -158.0,
+      "logps/rejected": -302.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.53125,
+      "rewards/margins": 19.0,
+      "rewards/rejected": -15.5,
+      "step": 1280
+    },
+    {
+      "epoch": 1.0336538461538463,
+      "grad_norm": 1.642467149992957e-05,
+      "learning_rate": 3.6420302760463045e-07,
+      "logits/chosen": 0.79296875,
+      "logits/rejected": 1.171875,
+      "logps/chosen": -163.0,
+      "logps/rejected": -292.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.46875,
+      "rewards/margins": 19.375,
+      "rewards/rejected": -15.9375,
+      "step": 1290
+    },
+    {
+      "epoch": 1.0416666666666667,
+      "grad_norm": 1.456498508279097e-05,
+      "learning_rate": 3.6271890768774116e-07,
+      "logits/chosen": 0.83984375,
+      "logits/rejected": 1.296875,
+      "logps/chosen": -138.0,
+      "logps/rejected": -276.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.390625,
+      "rewards/margins": 18.875,
+      "rewards/rejected": -15.4375,
+      "step": 1300
+    },
+    {
+      "epoch": 1.0496794871794872,
+      "grad_norm": 0.2184132961337863,
+      "learning_rate": 3.612347877708519e-07,
+      "logits/chosen": 0.796875,
+      "logits/rejected": 1.1015625,
+      "logps/chosen": -126.0,
+      "logps/rejected": -302.0,
+      "loss": 0.0013,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.59375,
+      "rewards/margins": 20.25,
+      "rewards/rejected": -16.625,
+      "step": 1310
+    },
+    {
+      "epoch": 1.0576923076923077,
+      "grad_norm": 0.00027060412424146214,
+      "learning_rate": 3.597506678539626e-07,
+      "logits/chosen": 0.78125,
+      "logits/rejected": 1.4375,
+      "logps/chosen": -162.0,
+      "logps/rejected": -294.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.78125,
+      "rewards/margins": 19.0,
+      "rewards/rejected": -15.1875,
+      "step": 1320
+    },
+    {
+      "epoch": 1.0657051282051282,
+      "grad_norm": 0.0005949930902462416,
+      "learning_rate": 3.582665479370733e-07,
+      "logits/chosen": 1.015625,
+      "logits/rejected": 1.1640625,
+      "logps/chosen": -133.0,
+      "logps/rejected": -280.0,
+      "loss": 0.0001,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.765625,
+      "rewards/margins": 18.625,
+      "rewards/rejected": -14.8125,
+      "step": 1330
+    },
+    {
+      "epoch": 1.0737179487179487,
+      "grad_norm": 3.821536446255942e-05,
+      "learning_rate": 3.5678242802018396e-07,
+      "logits/chosen": 0.98828125,
+      "logits/rejected": 1.0234375,
+      "logps/chosen": -169.0,
+      "logps/rejected": -284.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.8125,
+      "rewards/margins": 19.0,
+      "rewards/rejected": -15.25,
+      "step": 1340
+    },
+    {
+      "epoch": 1.0817307692307692,
+      "grad_norm": 0.0003595307379009215,
+      "learning_rate": 3.5529830810329473e-07,
+      "logits/chosen": 0.703125,
+      "logits/rejected": 1.109375,
+      "logps/chosen": -163.0,
+      "logps/rejected": -282.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.984375,
+      "rewards/margins": 18.875,
+      "rewards/rejected": -14.9375,
+      "step": 1350
+    },
+    {
+      "epoch": 1.0897435897435896,
+      "grad_norm": 0.21495162145805216,
+      "learning_rate": 3.5381418818640544e-07,
+      "logits/chosen": 0.828125,
+      "logits/rejected": 1.1796875,
+      "logps/chosen": -150.0,
+      "logps/rejected": -288.0,
+      "loss": 0.0192,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.421875,
+      "rewards/margins": 19.625,
+      "rewards/rejected": -16.25,
+      "step": 1360
+    },
+    {
+      "epoch": 1.0977564102564104,
+      "grad_norm": 0.00022987129229484713,
+      "learning_rate": 3.5233006826951616e-07,
+      "logits/chosen": 1.0390625,
+      "logits/rejected": 0.9921875,
+      "logps/chosen": -118.0,
+      "logps/rejected": -298.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.421875,
+      "rewards/margins": 20.0,
+      "rewards/rejected": -16.5,
+      "step": 1370
+    },
+    {
+      "epoch": 1.1057692307692308,
+      "grad_norm": 0.0001366686827476735,
+      "learning_rate": 3.5084594835262687e-07,
+      "logits/chosen": 0.80078125,
+      "logits/rejected": 0.98046875,
+      "logps/chosen": -125.0,
+      "logps/rejected": -284.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.546875,
+      "rewards/margins": 19.875,
+      "rewards/rejected": -16.375,
+      "step": 1380
+    },
+    {
+      "epoch": 1.1137820512820513,
+      "grad_norm": 1.2520041285180355e-05,
+      "learning_rate": 3.493618284357376e-07,
+      "logits/chosen": 0.90234375,
+      "logits/rejected": 1.3984375,
+      "logps/chosen": -168.0,
+      "logps/rejected": -312.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.4375,
+      "rewards/margins": 19.125,
+      "rewards/rejected": -15.75,
+      "step": 1390
+    },
+    {
+      "epoch": 1.1217948717948718,
+      "grad_norm": 0.002926505425137915,
+      "learning_rate": 3.478777085188483e-07,
+      "logits/chosen": 0.578125,
+      "logits/rejected": 0.921875,
+      "logps/chosen": -153.0,
+      "logps/rejected": -298.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.84375,
+      "rewards/margins": 20.0,
+      "rewards/rejected": -16.125,
+      "step": 1400
+    },
+    {
+      "epoch": 1.1298076923076923,
+      "grad_norm": 0.01017669683918559,
+      "learning_rate": 3.46393588601959e-07,
+      "logits/chosen": 0.94921875,
+      "logits/rejected": 1.125,
+      "logps/chosen": -126.0,
+      "logps/rejected": -308.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.46875,
+      "rewards/margins": 21.125,
+      "rewards/rejected": -17.625,
+      "step": 1410
+    },
+    {
+      "epoch": 1.1378205128205128,
+      "grad_norm": 4.668022191357913e-06,
+      "learning_rate": 3.449094686850698e-07,
+      "logits/chosen": 0.8984375,
+      "logits/rejected": 1.3984375,
+      "logps/chosen": -155.0,
+      "logps/rejected": -314.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.984375,
+      "rewards/margins": 20.75,
+      "rewards/rejected": -16.75,
+      "step": 1420
+    },
+    {
+      "epoch": 1.1458333333333333,
+      "grad_norm": 4.630494393827752e-05,
+      "learning_rate": 3.434253487681805e-07,
+      "logits/chosen": 0.671875,
+      "logits/rejected": 0.94140625,
+      "logps/chosen": -177.0,
+      "logps/rejected": -296.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.53125,
+      "rewards/margins": 19.375,
+      "rewards/rejected": -15.8125,
+      "step": 1430
+    },
+    {
+      "epoch": 1.1538461538461537,
+      "grad_norm": 0.0006577813730898143,
+      "learning_rate": 3.4194122885129115e-07,
+      "logits/chosen": 0.73828125,
+      "logits/rejected": 0.94921875,
+      "logps/chosen": -180.0,
+      "logps/rejected": -282.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.6875,
+      "rewards/margins": 18.125,
+      "rewards/rejected": -14.4375,
+      "step": 1440
+    },
+    {
+      "epoch": 1.1618589743589745,
+      "grad_norm": 0.0004865147564756555,
+      "learning_rate": 3.4045710893440187e-07,
+      "logits/chosen": 0.66796875,
+      "logits/rejected": 1.1640625,
+      "logps/chosen": -170.0,
+      "logps/rejected": -304.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.65625,
+      "rewards/margins": 20.75,
+      "rewards/rejected": -17.125,
+      "step": 1450
+    },
+    {
+      "epoch": 1.169871794871795,
+      "grad_norm": 8.243889755055517e-05,
+      "learning_rate": 3.389729890175126e-07,
+      "logits/chosen": 0.73828125,
+      "logits/rejected": 1.1171875,
+      "logps/chosen": -174.0,
+      "logps/rejected": -300.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 4.03125,
+      "rewards/margins": 20.25,
+      "rewards/rejected": -16.25,
+      "step": 1460
+    },
+    {
+      "epoch": 1.1778846153846154,
+      "grad_norm": 0.055297406196161324,
+      "learning_rate": 3.374888691006233e-07,
+      "logits/chosen": 0.68359375,
+      "logits/rejected": 1.0078125,
+      "logps/chosen": -180.0,
+      "logps/rejected": -292.0,
+      "loss": 0.0001,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.9375,
+      "rewards/margins": 19.875,
+      "rewards/rejected": -15.9375,
+      "step": 1470
+    },
+    {
+      "epoch": 1.185897435897436,
+      "grad_norm": 3.940056875422956e-06,
+      "learning_rate": 3.36004749183734e-07,
+      "logits/chosen": 0.7734375,
+      "logits/rejected": 1.1875,
+      "logps/chosen": -187.0,
+      "logps/rejected": -282.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.8125,
+      "rewards/margins": 19.0,
+      "rewards/rejected": -15.1875,
+      "step": 1480
+    },
+    {
+      "epoch": 1.1939102564102564,
+      "grad_norm": 0.02924640642898095,
+      "learning_rate": 3.3452062926684477e-07,
+      "logits/chosen": 0.77734375,
+      "logits/rejected": 0.87890625,
+      "logps/chosen": -151.0,
+      "logps/rejected": -294.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.359375,
+      "rewards/margins": 19.625,
+      "rewards/rejected": -16.25,
+      "step": 1490
+    },
+    {
+      "epoch": 1.2019230769230769,
+      "grad_norm": 0.003264557976511741,
+      "learning_rate": 3.330365093499555e-07,
+      "logits/chosen": 0.7109375,
+      "logits/rejected": 0.8828125,
+      "logps/chosen": -181.0,
+      "logps/rejected": -280.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.96875,
+      "rewards/margins": 18.75,
+      "rewards/rejected": -14.75,
+      "step": 1500
+    },
+    {
+      "epoch": 1.2099358974358974,
+      "grad_norm": 1.5994308395955108e-05,
+      "learning_rate": 3.315523894330662e-07,
+      "logits/chosen": 0.63671875,
+      "logits/rejected": 0.77734375,
+      "logps/chosen": -180.0,
+      "logps/rejected": -306.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.703125,
+      "rewards/margins": 20.625,
+      "rewards/rejected": -16.875,
+      "step": 1510
+    },
+    {
+      "epoch": 1.217948717948718,
+      "grad_norm": 0.0006042891458313199,
+      "learning_rate": 3.300682695161769e-07,
+      "logits/chosen": 0.6484375,
+      "logits/rejected": 1.328125,
+      "logps/chosen": -158.0,
+      "logps/rejected": -294.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.765625,
+      "rewards/margins": 19.625,
+      "rewards/rejected": -15.9375,
+      "step": 1520
+    },
+    {
+      "epoch": 1.2259615384615385,
+      "grad_norm": 0.00021178126794762857,
+      "learning_rate": 3.2858414959928757e-07,
+      "logits/chosen": 0.65625,
+      "logits/rejected": 1.078125,
+      "logps/chosen": -116.0,
+      "logps/rejected": -290.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.265625,
+      "rewards/margins": 20.625,
+      "rewards/rejected": -17.375,
+      "step": 1530
+    },
+    {
+      "epoch": 1.233974358974359,
+      "grad_norm": 0.0010273708366364401,
+      "learning_rate": 3.271000296823983e-07,
+      "logits/chosen": 0.65625,
+      "logits/rejected": 1.375,
+      "logps/chosen": -152.0,
+      "logps/rejected": -308.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.28125,
+      "rewards/margins": 20.25,
+      "rewards/rejected": -17.0,
+      "step": 1540
+    },
+    {
+      "epoch": 1.2419871794871795,
+      "grad_norm": 3.07402349065646e-06,
+      "learning_rate": 3.25615909765509e-07,
+      "logits/chosen": 0.921875,
+      "logits/rejected": 1.4453125,
+      "logps/chosen": -142.0,
+      "logps/rejected": -304.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.171875,
+      "rewards/margins": 20.125,
+      "rewards/rejected": -16.875,
+      "step": 1550
+    },
+    {
+      "epoch": 1.25,
+      "grad_norm": 0.0011000584431525772,
+      "learning_rate": 3.2413178984861977e-07,
+      "logits/chosen": 0.70703125,
+      "logits/rejected": 1.0390625,
+      "logps/chosen": -117.0,
+      "logps/rejected": -292.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.578125,
+      "rewards/margins": 20.0,
+      "rewards/rejected": -16.375,
+      "step": 1560
+    },
+    {
+      "epoch": 1.2580128205128205,
+      "grad_norm": 0.000452369898432101,
+      "learning_rate": 3.226476699317305e-07,
+      "logits/chosen": 0.671875,
+      "logits/rejected": 1.0703125,
+      "logps/chosen": -150.0,
+      "logps/rejected": -316.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.796875,
+      "rewards/margins": 21.25,
+      "rewards/rejected": -17.375,
+      "step": 1570
+    },
+    {
+      "epoch": 1.266025641025641,
+      "grad_norm": 0.00026586619030460033,
+      "learning_rate": 3.211635500148412e-07,
+      "logits/chosen": 0.8671875,
+      "logits/rejected": 1.1640625,
+      "logps/chosen": -160.0,
+      "logps/rejected": -306.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.578125,
+      "rewards/margins": 20.25,
+      "rewards/rejected": -16.75,
+      "step": 1580
+    },
+    {
+      "epoch": 1.2740384615384617,
+      "grad_norm": 2.7345902360690067e-06,
+      "learning_rate": 3.196794300979519e-07,
+      "logits/chosen": 0.76171875,
+      "logits/rejected": 1.140625,
+      "logps/chosen": -185.0,
+      "logps/rejected": -312.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.65625,
+      "rewards/margins": 20.75,
+      "rewards/rejected": -17.125,
+      "step": 1590
+    },
+    {
+      "epoch": 1.282051282051282,
+      "grad_norm": 1.5071041983034111e-06,
+      "learning_rate": 3.181953101810626e-07,
+      "logits/chosen": 0.53515625,
+      "logits/rejected": 0.96875,
+      "logps/chosen": -153.0,
+      "logps/rejected": -300.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.90625,
+      "rewards/margins": 20.375,
+      "rewards/rejected": -16.5,
+      "step": 1600
+    },
+    {
+      "epoch": 1.2900641025641026,
+      "grad_norm": 9.809509824262037e-05,
+      "learning_rate": 3.1671119026417333e-07,
+      "logits/chosen": 0.71875,
+      "logits/rejected": 0.9921875,
+      "logps/chosen": -145.0,
+      "logps/rejected": -302.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.203125,
+      "rewards/margins": 19.625,
+      "rewards/rejected": -16.375,
+      "step": 1610
+    },
+    {
+      "epoch": 1.2980769230769231,
+      "grad_norm": 0.00019153122395682397,
+      "learning_rate": 3.15227070347284e-07,
+      "logits/chosen": 0.8828125,
+      "logits/rejected": 1.1015625,
+      "logps/chosen": -159.0,
+      "logps/rejected": -292.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.96875,
+      "rewards/margins": 19.5,
+      "rewards/rejected": -15.625,
+      "step": 1620
+    },
+    {
+      "epoch": 1.3060897435897436,
+      "grad_norm": 1.1285816682202219e-06,
+      "learning_rate": 3.1374295043039476e-07,
+      "logits/chosen": 0.91015625,
+      "logits/rejected": 1.359375,
+      "logps/chosen": -139.0,
+      "logps/rejected": -304.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.46875,
+      "rewards/margins": 21.75,
+      "rewards/rejected": -18.25,
+      "step": 1630
+    },
+    {
+      "epoch": 1.314102564102564,
+      "grad_norm": 0.00014596821162342946,
+      "learning_rate": 3.122588305135055e-07,
+      "logits/chosen": 0.8984375,
+      "logits/rejected": 0.99609375,
+      "logps/chosen": -164.0,
+      "logps/rejected": -300.0,
+      "loss": 0.0003,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.0625,
+      "rewards/margins": 19.625,
+      "rewards/rejected": -16.5,
+      "step": 1640
+    },
+    {
+      "epoch": 1.3221153846153846,
+      "grad_norm": 6.180429625577289e-05,
+      "learning_rate": 3.107747105966162e-07,
+      "logits/chosen": 0.5859375,
+      "logits/rejected": 0.91015625,
+      "logps/chosen": -140.0,
+      "logps/rejected": -304.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.6875,
+      "rewards/margins": 21.5,
+      "rewards/rejected": -17.75,
+      "step": 1650
+    },
+    {
+      "epoch": 1.330128205128205,
+      "grad_norm": 1.528718002747695e-05,
+      "learning_rate": 3.092905906797269e-07,
+      "logits/chosen": 0.96484375,
+      "logits/rejected": 1.25,
+      "logps/chosen": -160.0,
+      "logps/rejected": -312.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.59375,
+      "rewards/margins": 21.125,
+      "rewards/rejected": -17.5,
+      "step": 1660
+    },
+    {
+      "epoch": 1.3381410256410255,
+      "grad_norm": 0.0018059387091916529,
+      "learning_rate": 3.078064707628376e-07,
+      "logits/chosen": 0.96484375,
+      "logits/rejected": 1.3515625,
+      "logps/chosen": -150.0,
+      "logps/rejected": -298.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.265625,
+      "rewards/margins": 20.875,
+      "rewards/rejected": -17.625,
+      "step": 1670
+    },
+    {
+      "epoch": 1.3461538461538463,
+      "grad_norm": 9.308236631197583e-05,
+      "learning_rate": 3.0632235084594833e-07,
+      "logits/chosen": 0.94140625,
+      "logits/rejected": 1.21875,
+      "logps/chosen": -154.0,
+      "logps/rejected": -298.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.59375,
+      "rewards/margins": 20.25,
+      "rewards/rejected": -16.75,
+      "step": 1680
+    },
+    {
+      "epoch": 1.3541666666666667,
+      "grad_norm": 1.0292832754568623e-05,
+      "learning_rate": 3.048382309290591e-07,
+      "logits/chosen": 0.9453125,
+      "logits/rejected": 1.1953125,
+      "logps/chosen": -149.0,
+      "logps/rejected": -314.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.578125,
+      "rewards/margins": 21.0,
+      "rewards/rejected": -17.375,
+      "step": 1690
+    },
+    {
+      "epoch": 1.3621794871794872,
+      "grad_norm": 0.002936471913731386,
+      "learning_rate": 3.033541110121698e-07,
+      "logits/chosen": 0.67578125,
+      "logits/rejected": 0.8515625,
+      "logps/chosen": -178.0,
+      "logps/rejected": -304.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.609375,
+      "rewards/margins": 20.75,
+      "rewards/rejected": -17.125,
+      "step": 1700
+    },
+    {
+      "epoch": 1.3701923076923077,
+      "grad_norm": 0.00010907511941114075,
+      "learning_rate": 3.018699910952805e-07,
+      "logits/chosen": 0.796875,
+      "logits/rejected": 0.9609375,
+      "logps/chosen": -171.0,
+      "logps/rejected": -296.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.625,
+      "rewards/margins": 19.375,
+      "rewards/rejected": -15.75,
+      "step": 1710
+    },
+    {
+      "epoch": 1.3782051282051282,
+      "grad_norm": 0.004723298049564672,
+      "learning_rate": 3.003858711783912e-07,
+      "logits/chosen": 0.93359375,
+      "logits/rejected": 1.34375,
+      "logps/chosen": -128.0,
+      "logps/rejected": -304.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.40625,
+      "rewards/margins": 20.75,
+      "rewards/rejected": -17.375,
+      "step": 1720
+    },
+    {
+      "epoch": 1.3862179487179487,
+      "grad_norm": 0.0037479058484858564,
+      "learning_rate": 2.989017512615019e-07,
+      "logits/chosen": 0.81640625,
+      "logits/rejected": 1.3359375,
+      "logps/chosen": -187.0,
+      "logps/rejected": -316.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.84375,
+      "rewards/margins": 20.625,
+      "rewards/rejected": -16.75,
+      "step": 1730
+    },
+    {
+      "epoch": 1.3942307692307692,
+      "grad_norm": 6.403847125969602e-06,
+      "learning_rate": 2.974176313446126e-07,
+      "logits/chosen": 0.8984375,
+      "logits/rejected": 1.40625,
+      "logps/chosen": -167.0,
+      "logps/rejected": -304.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.46875,
+      "rewards/margins": 20.5,
+      "rewards/rejected": -17.125,
+      "step": 1740
+    },
+    {
+      "epoch": 1.4022435897435899,
+      "grad_norm": 3.170124782559705e-06,
+      "learning_rate": 2.959335114277233e-07,
+      "logits/chosen": 0.4765625,
+      "logits/rejected": 0.98046875,
+      "logps/chosen": -200.0,
+      "logps/rejected": -312.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.71875,
+      "rewards/margins": 20.75,
+      "rewards/rejected": -17.0,
+      "step": 1750
+    },
+    {
+      "epoch": 1.4102564102564101,
+      "grad_norm": 0.002625958073479466,
+      "learning_rate": 2.944493915108341e-07,
+      "logits/chosen": 0.8125,
+      "logits/rejected": 0.984375,
+      "logps/chosen": -145.0,
+      "logps/rejected": -288.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.375,
+      "rewards/margins": 19.375,
+      "rewards/rejected": -16.0,
+      "step": 1760
+    },
+    {
+      "epoch": 1.4182692307692308,
+      "grad_norm": 1.7345808005103415e-05,
+      "learning_rate": 2.929652715939448e-07,
+      "logits/chosen": 0.66015625,
+      "logits/rejected": 1.203125,
+      "logps/chosen": -186.0,
+      "logps/rejected": -304.0,
+      "loss": 0.0008,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.53125,
+      "rewards/margins": 20.875,
+      "rewards/rejected": -17.375,
+      "step": 1770
+    },
+    {
+      "epoch": 1.4262820512820513,
+      "grad_norm": 3.70136218823452e-06,
+      "learning_rate": 2.914811516770555e-07,
+      "logits/chosen": 0.486328125,
+      "logits/rejected": 1.0078125,
+      "logps/chosen": -209.0,
+      "logps/rejected": -320.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.59375,
+      "rewards/margins": 20.875,
+      "rewards/rejected": -17.375,
+      "step": 1780
+    },
+    {
+      "epoch": 1.4342948717948718,
+      "grad_norm": 1.1884302830057285e-05,
+      "learning_rate": 2.8999703176016623e-07,
+      "logits/chosen": 0.8046875,
+      "logits/rejected": 1.34375,
+      "logps/chosen": -176.0,
+      "logps/rejected": -296.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.0625,
+      "rewards/margins": 19.625,
+      "rewards/rejected": -16.625,
+      "step": 1790
+    },
+    {
+      "epoch": 1.4423076923076923,
+      "grad_norm": 0.0004609700315428155,
+      "learning_rate": 2.8851291184327694e-07,
+      "logits/chosen": 0.703125,
+      "logits/rejected": 1.09375,
+      "logps/chosen": -164.0,
+      "logps/rejected": -322.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.078125,
+      "rewards/margins": 22.125,
+      "rewards/rejected": -19.125,
+      "step": 1800
+    },
+    {
+      "epoch": 1.4503205128205128,
+      "grad_norm": 0.0006305642233578017,
+      "learning_rate": 2.870287919263876e-07,
+      "logits/chosen": 0.72265625,
+      "logits/rejected": 1.1953125,
+      "logps/chosen": -185.0,
+      "logps/rejected": -312.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.875,
+      "rewards/margins": 20.875,
+      "rewards/rejected": -18.0,
+      "step": 1810
+    },
+    {
+      "epoch": 1.4583333333333333,
+      "grad_norm": 3.4339114874993125e-06,
+      "learning_rate": 2.855446720094983e-07,
+      "logits/chosen": 0.7421875,
+      "logits/rejected": 1.3203125,
+      "logps/chosen": -165.0,
+      "logps/rejected": -316.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.75,
+      "rewards/margins": 21.25,
+      "rewards/rejected": -17.5,
+      "step": 1820
+    },
+    {
+      "epoch": 1.4663461538461537,
+      "grad_norm": 0.002348896026755423,
+      "learning_rate": 2.840605520926091e-07,
+      "logits/chosen": 0.7265625,
+      "logits/rejected": 0.98046875,
+      "logps/chosen": -181.0,
+      "logps/rejected": -304.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.25,
+      "rewards/margins": 20.875,
+      "rewards/rejected": -17.625,
+      "step": 1830
+    },
+    {
+      "epoch": 1.4743589743589745,
+      "grad_norm": 4.445170925547984e-05,
+      "learning_rate": 2.825764321757198e-07,
+      "logits/chosen": 0.78515625,
+      "logits/rejected": 0.93359375,
+      "logps/chosen": -172.0,
+      "logps/rejected": -302.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.515625,
+      "rewards/margins": 20.875,
+      "rewards/rejected": -17.375,
+      "step": 1840
+    },
+    {
+      "epoch": 1.482371794871795,
+      "grad_norm": 0.16403277766161206,
+      "learning_rate": 2.810923122588305e-07,
+      "logits/chosen": 0.7265625,
+      "logits/rejected": 1.0859375,
+      "logps/chosen": -162.0,
+      "logps/rejected": -300.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.390625,
+      "rewards/margins": 19.75,
+      "rewards/rejected": -16.375,
+      "step": 1850
+    },
+    {
+      "epoch": 1.4903846153846154,
+      "grad_norm": 0.0008420288269927029,
+      "learning_rate": 2.796081923419412e-07,
+      "logits/chosen": 0.71875,
+      "logits/rejected": 1.3125,
+      "logps/chosen": -197.0,
+      "logps/rejected": -324.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.4375,
+      "rewards/margins": 21.5,
+      "rewards/rejected": -18.0,
+      "step": 1860
+    },
+    {
+      "epoch": 1.498397435897436,
+      "grad_norm": 0.00014053022075140328,
+      "learning_rate": 2.7812407242505194e-07,
+      "logits/chosen": 0.9921875,
+      "logits/rejected": 1.515625,
+      "logps/chosen": -160.0,
+      "logps/rejected": -328.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.828125,
+      "rewards/margins": 23.0,
+      "rewards/rejected": -19.125,
+      "step": 1870
+    },
+    {
+      "epoch": 1.5064102564102564,
+      "grad_norm": 8.433668468764786e-05,
+      "learning_rate": 2.7663995250816265e-07,
+      "logits/chosen": 1.0390625,
+      "logits/rejected": 1.09375,
+      "logps/chosen": -150.0,
+      "logps/rejected": -326.0,
+      "loss": 0.0001,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.21875,
+      "rewards/margins": 22.25,
+      "rewards/rejected": -19.0,
+      "step": 1880
+    },
+    {
+      "epoch": 1.5144230769230769,
+      "grad_norm": 3.8086614833106377e-06,
+      "learning_rate": 2.7515583259127337e-07,
+      "logits/chosen": 0.5,
+      "logits/rejected": 1.0234375,
+      "logps/chosen": -216.0,
+      "logps/rejected": -316.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.765625,
+      "rewards/margins": 22.125,
+      "rewards/rejected": -18.375,
+      "step": 1890
+    },
+    {
+      "epoch": 1.5224358974358974,
+      "grad_norm": 5.226952675441339e-07,
+      "learning_rate": 2.736717126743841e-07,
+      "logits/chosen": 0.72265625,
+      "logits/rejected": 1.0859375,
+      "logps/chosen": -165.0,
+      "logps/rejected": -334.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.703125,
+      "rewards/margins": 22.75,
+      "rewards/rejected": -19.0,
+      "step": 1900
+    },
+    {
+      "epoch": 1.530448717948718,
+      "grad_norm": 2.2091624737157388e-05,
+      "learning_rate": 2.721875927574948e-07,
+      "logits/chosen": 0.75,
+      "logits/rejected": 1.2578125,
+      "logps/chosen": -189.0,
+      "logps/rejected": -316.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.84375,
+      "rewards/margins": 22.125,
+      "rewards/rejected": -18.25,
+      "step": 1910
+    },
+    {
+      "epoch": 1.5384615384615383,
+      "grad_norm": 0.00010413244451900578,
+      "learning_rate": 2.707034728406055e-07,
+      "logits/chosen": 0.76171875,
+      "logits/rejected": 1.265625,
+      "logps/chosen": -182.0,
+      "logps/rejected": -312.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.6875,
+      "rewards/margins": 21.5,
+      "rewards/rejected": -17.75,
+      "step": 1920
+    },
+    {
+      "epoch": 1.546474358974359,
+      "grad_norm": 5.596299324175904e-05,
+      "learning_rate": 2.692193529237162e-07,
+      "logits/chosen": 0.90234375,
+      "logits/rejected": 0.984375,
+      "logps/chosen": -145.0,
+      "logps/rejected": -328.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.515625,
+      "rewards/margins": 23.375,
+      "rewards/rejected": -19.875,
+      "step": 1930
+    },
+    {
+      "epoch": 1.5544871794871795,
+      "grad_norm": 9.505443984664321e-07,
+      "learning_rate": 2.6773523300682693e-07,
+      "logits/chosen": 0.8828125,
+      "logits/rejected": 1.28125,
+      "logps/chosen": -150.0,
+      "logps/rejected": -306.0,
+      "loss": 0.0029,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.15625,
+      "rewards/margins": 20.875,
+      "rewards/rejected": -17.75,
+      "step": 1940
+    },
+    {
+      "epoch": 1.5625,
+      "grad_norm": 7.9835406414024e-05,
+      "learning_rate": 2.6625111308993765e-07,
+      "logits/chosen": 0.78125,
+      "logits/rejected": 1.25,
+      "logps/chosen": -155.0,
+      "logps/rejected": -310.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.65625,
+      "rewards/margins": 22.125,
+      "rewards/rejected": -18.5,
+      "step": 1950
+    },
+    {
+      "epoch": 1.5705128205128205,
+      "grad_norm": 5.207840607481746e-06,
+      "learning_rate": 2.6476699317304836e-07,
+      "logits/chosen": 0.77734375,
+      "logits/rejected": 1.2734375,
+      "logps/chosen": -197.0,
+      "logps/rejected": -308.0,
+      "loss": 0.0001,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.484375,
+      "rewards/margins": 21.25,
+      "rewards/rejected": -17.75,
+      "step": 1960
+    },
+    {
+      "epoch": 1.578525641025641,
+      "grad_norm": 0.0055069480205136,
+      "learning_rate": 2.6328287325615913e-07,
+      "logits/chosen": 0.55859375,
+      "logits/rejected": 0.9765625,
+      "logps/chosen": -181.0,
+      "logps/rejected": -310.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.5625,
+      "rewards/margins": 20.875,
+      "rewards/rejected": -17.25,
+      "step": 1970
+    },
+    {
+      "epoch": 1.5865384615384617,
+      "grad_norm": 1.396497021073787e-05,
+      "learning_rate": 2.6179875333926984e-07,
+      "logits/chosen": 0.890625,
+      "logits/rejected": 1.0546875,
+      "logps/chosen": -162.0,
+      "logps/rejected": -330.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.890625,
+      "rewards/margins": 23.375,
+      "rewards/rejected": -19.5,
+      "step": 1980
+    },
+    {
+      "epoch": 1.594551282051282,
+      "grad_norm": 0.0017364799756809,
+      "learning_rate": 2.603146334223805e-07,
+      "logits/chosen": 0.74609375,
+      "logits/rejected": 1.09375,
+      "logps/chosen": -167.0,
+      "logps/rejected": -302.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.296875,
+      "rewards/margins": 20.5,
+      "rewards/rejected": -17.25,
+      "step": 1990
+    },
+    {
+      "epoch": 1.6025641025641026,
+      "grad_norm": 1.0811679994728376e-05,
+      "learning_rate": 2.588305135054912e-07,
+      "logits/chosen": 0.7890625,
+      "logits/rejected": 1.0390625,
+      "logps/chosen": -190.0,
+      "logps/rejected": -322.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 4.3125,
+      "rewards/margins": 23.125,
+      "rewards/rejected": -18.75,
+      "step": 2000
+    },
+    {
+      "epoch": 1.6105769230769231,
+      "grad_norm": 1.0141558630107686e-07,
+      "learning_rate": 2.5734639358860193e-07,
+      "logits/chosen": 0.80078125,
+      "logits/rejected": 1.421875,
+      "logps/chosen": -174.0,
+      "logps/rejected": -316.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.765625,
+      "rewards/margins": 21.625,
+      "rewards/rejected": -17.875,
+      "step": 2010
+    },
+    {
+      "epoch": 1.6185897435897436,
+      "grad_norm": 3.9088308067055224e-06,
+      "learning_rate": 2.5586227367171264e-07,
+      "logits/chosen": 0.86328125,
+      "logits/rejected": 1.15625,
+      "logps/chosen": -163.0,
+      "logps/rejected": -302.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.28125,
+      "rewards/margins": 21.0,
+      "rewards/rejected": -17.75,
+      "step": 2020
+    },
+    {
+      "epoch": 1.626602564102564,
+      "grad_norm": 4.13722853892595e-06,
+      "learning_rate": 2.5437815375482335e-07,
+      "logits/chosen": 0.84375,
+      "logits/rejected": 1.0703125,
+      "logps/chosen": -168.0,
+      "logps/rejected": -338.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.25,
+      "rewards/margins": 22.0,
+      "rewards/rejected": -18.75,
+      "step": 2030
+    },
+    {
+      "epoch": 1.6346153846153846,
+      "grad_norm": 3.401296459983001e-05,
+      "learning_rate": 2.528940338379341e-07,
+      "logits/chosen": 1.0234375,
+      "logits/rejected": 1.1875,
+      "logps/chosen": -159.0,
+      "logps/rejected": -326.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.859375,
+      "rewards/margins": 23.0,
+      "rewards/rejected": -19.125,
+      "step": 2040
+    },
+    {
+      "epoch": 1.6426282051282053,
+      "grad_norm": 2.957293101803746e-07,
+      "learning_rate": 2.5140991392104483e-07,
+      "logits/chosen": 0.87109375,
+      "logits/rejected": 1.28125,
+      "logps/chosen": -97.0,
+      "logps/rejected": -328.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.203125,
+      "rewards/margins": 22.625,
+      "rewards/rejected": -19.375,
+      "step": 2050
+    },
+    {
+      "epoch": 1.6506410256410255,
+      "grad_norm": 5.9562249969748895e-06,
+      "learning_rate": 2.4992579400415555e-07,
+      "logits/chosen": 0.796875,
+      "logits/rejected": 1.0390625,
+      "logps/chosen": -165.0,
+      "logps/rejected": -296.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.71875,
+      "rewards/margins": 21.25,
+      "rewards/rejected": -17.5,
+      "step": 2060
+    },
+    {
+      "epoch": 1.6586538461538463,
+      "grad_norm": 6.175150368480905e-06,
+      "learning_rate": 2.4844167408726626e-07,
+      "logits/chosen": 0.984375,
+      "logits/rejected": 1.4375,
+      "logps/chosen": -160.0,
+      "logps/rejected": -324.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.8125,
+      "rewards/margins": 22.125,
+      "rewards/rejected": -18.25,
+      "step": 2070
+    },
+    {
+      "epoch": 1.6666666666666665,
+      "grad_norm": 2.6561524482376626e-05,
+      "learning_rate": 2.469575541703769e-07,
+      "logits/chosen": 0.83203125,
+      "logits/rejected": 1.1796875,
+      "logps/chosen": -169.0,
+      "logps/rejected": -322.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.546875,
+      "rewards/margins": 22.625,
+      "rewards/rejected": -19.0,
+      "step": 2080
+    },
+    {
+      "epoch": 1.6746794871794872,
+      "grad_norm": 4.068151187669672e-06,
+      "learning_rate": 2.454734342534877e-07,
+      "logits/chosen": 0.96875,
+      "logits/rejected": 1.171875,
+      "logps/chosen": -184.0,
+      "logps/rejected": -298.0,
+      "loss": 0.0034,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.703125,
+      "rewards/margins": 20.625,
+      "rewards/rejected": -16.875,
+      "step": 2090
+    },
+    {
+      "epoch": 1.6826923076923077,
+      "grad_norm": 2.6204722144054077e-05,
+      "learning_rate": 2.439893143365984e-07,
+      "logits/chosen": 1.015625,
+      "logits/rejected": 1.296875,
+      "logps/chosen": -155.0,
+      "logps/rejected": -314.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.625,
+      "rewards/margins": 22.25,
+      "rewards/rejected": -18.625,
+      "step": 2100
+    },
+    {
+      "epoch": 1.6907051282051282,
+      "grad_norm": 2.398210733527027e-06,
+      "learning_rate": 2.425051944197091e-07,
+      "logits/chosen": 0.8984375,
+      "logits/rejected": 1.1640625,
+      "logps/chosen": -142.0,
+      "logps/rejected": -338.0,
+      "loss": 0.0002,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.390625,
+      "rewards/margins": 23.375,
+      "rewards/rejected": -20.0,
+      "step": 2110
+    },
+    {
+      "epoch": 1.6987179487179487,
+      "grad_norm": 4.12694708820085e-06,
+      "learning_rate": 2.4102107450281983e-07,
+      "logits/chosen": 0.8984375,
+      "logits/rejected": 1.3359375,
+      "logps/chosen": -160.0,
+      "logps/rejected": -308.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.03125,
+      "rewards/margins": 19.75,
+      "rewards/rejected": -16.625,
+      "step": 2120
+    },
+    {
+      "epoch": 1.7067307692307692,
+      "grad_norm": 0.00013371742562856578,
+      "learning_rate": 2.3953695458593054e-07,
+      "logits/chosen": 0.86328125,
+      "logits/rejected": 1.109375,
+      "logps/chosen": -146.0,
+      "logps/rejected": -328.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.75,
+      "rewards/margins": 23.75,
+      "rewards/rejected": -19.875,
+      "step": 2130
+    },
+    {
+      "epoch": 1.7147435897435899,
+      "grad_norm": 7.636681790511307e-06,
+      "learning_rate": 2.3805283466904126e-07,
+      "logits/chosen": 1.0078125,
+      "logits/rejected": 1.1796875,
+      "logps/chosen": -166.0,
+      "logps/rejected": -310.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.8125,
+      "rewards/margins": 21.625,
+      "rewards/rejected": -17.75,
+      "step": 2140
+    },
+    {
+      "epoch": 1.7227564102564101,
+      "grad_norm": 8.958009719967443e-07,
+      "learning_rate": 2.3656871475215194e-07,
+      "logits/chosen": 0.70703125,
+      "logits/rejected": 1.1875,
+      "logps/chosen": -154.0,
+      "logps/rejected": -338.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.28125,
+      "rewards/margins": 22.5,
+      "rewards/rejected": -19.25,
+      "step": 2150
+    },
+    {
+      "epoch": 1.7307692307692308,
+      "grad_norm": 3.747660209974031e-05,
+      "learning_rate": 2.3508459483526268e-07,
+      "logits/chosen": 0.9453125,
+      "logits/rejected": 1.390625,
+      "logps/chosen": -186.0,
+      "logps/rejected": -312.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.84375,
+      "rewards/margins": 22.125,
+      "rewards/rejected": -18.25,
+      "step": 2160
+    },
+    {
+      "epoch": 1.7387820512820513,
+      "grad_norm": 4.869196145111484e-07,
+      "learning_rate": 2.336004749183734e-07,
+      "logits/chosen": 0.81640625,
+      "logits/rejected": 1.140625,
+      "logps/chosen": -149.0,
+      "logps/rejected": -320.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.328125,
+      "rewards/margins": 22.75,
+      "rewards/rejected": -19.375,
+      "step": 2170
+    },
+    {
+      "epoch": 1.7467948717948718,
+      "grad_norm": 0.00026582023320058276,
+      "learning_rate": 2.321163550014841e-07,
+      "logits/chosen": 0.796875,
+      "logits/rejected": 1.3046875,
+      "logps/chosen": -133.0,
+      "logps/rejected": -336.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.15625,
+      "rewards/margins": 22.75,
+      "rewards/rejected": -19.625,
+      "step": 2180
+    },
+    {
+      "epoch": 1.7548076923076923,
+      "grad_norm": 2.8032704053085966e-06,
+      "learning_rate": 2.3063223508459482e-07,
+      "logits/chosen": 0.8125,
+      "logits/rejected": 1.1171875,
+      "logps/chosen": -172.0,
+      "logps/rejected": -318.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.375,
+      "rewards/margins": 22.375,
+      "rewards/rejected": -19.0,
+      "step": 2190
+    },
+    {
+      "epoch": 1.7628205128205128,
+      "grad_norm": 0.0028628851089617393,
+      "learning_rate": 2.2914811516770554e-07,
+      "logits/chosen": 0.859375,
+      "logits/rejected": 1.1796875,
+      "logps/chosen": -185.0,
+      "logps/rejected": -300.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.25,
+      "rewards/margins": 19.875,
+      "rewards/rejected": -16.625,
+      "step": 2200
+    },
+    {
+      "epoch": 1.7708333333333335,
+      "grad_norm": 0.0001342614273603189,
+      "learning_rate": 2.2766399525081625e-07,
+      "logits/chosen": 1.0859375,
+      "logits/rejected": 1.1640625,
+      "logps/chosen": -177.0,
+      "logps/rejected": -320.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.875,
+      "rewards/margins": 21.75,
+      "rewards/rejected": -17.875,
+      "step": 2210
+    },
+    {
+      "epoch": 1.7788461538461537,
+      "grad_norm": 4.955335994263196e-07,
+      "learning_rate": 2.2617987533392696e-07,
+      "logits/chosen": 0.828125,
+      "logits/rejected": 1.21875,
+      "logps/chosen": -158.0,
+      "logps/rejected": -318.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.25,
+      "rewards/margins": 21.5,
+      "rewards/rejected": -18.25,
+      "step": 2220
+    },
+    {
+      "epoch": 1.7868589743589745,
+      "grad_norm": 0.0014087673231885866,
+      "learning_rate": 2.246957554170377e-07,
+      "logits/chosen": 0.7265625,
+      "logits/rejected": 1.03125,
+      "logps/chosen": -188.0,
+      "logps/rejected": -316.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.40625,
+      "rewards/margins": 21.5,
+      "rewards/rejected": -18.125,
+      "step": 2230
+    },
+    {
+      "epoch": 1.7948717948717947,
+      "grad_norm": 0.00019680490630025088,
+      "learning_rate": 2.232116355001484e-07,
+      "logits/chosen": 0.7578125,
+      "logits/rejected": 1.1875,
+      "logps/chosen": -167.0,
+      "logps/rejected": -326.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.21875,
+      "rewards/margins": 21.25,
+      "rewards/rejected": -18.0,
+      "step": 2240
+    },
+    {
+      "epoch": 1.8028846153846154,
+      "grad_norm": 1.3924207072166965e-06,
+      "learning_rate": 2.217275155832591e-07,
+      "logits/chosen": 0.72265625,
+      "logits/rejected": 0.98046875,
+      "logps/chosen": -162.0,
+      "logps/rejected": -336.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 4.03125,
+      "rewards/margins": 23.75,
+      "rewards/rejected": -19.625,
+      "step": 2250
+    },
+    {
+      "epoch": 1.810897435897436,
+      "grad_norm": 2.7880513613496112e-05,
+      "learning_rate": 2.2024339566636982e-07,
+      "logits/chosen": 1.046875,
+      "logits/rejected": 1.3671875,
+      "logps/chosen": -152.0,
+      "logps/rejected": -288.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.796875,
+      "rewards/margins": 19.875,
+      "rewards/rejected": -16.125,
+      "step": 2260
+    },
+    {
+      "epoch": 1.8189102564102564,
+      "grad_norm": 2.01578072742099e-05,
+      "learning_rate": 2.1875927574948056e-07,
+      "logits/chosen": 0.9375,
+      "logits/rejected": 1.015625,
+      "logps/chosen": -170.0,
+      "logps/rejected": -328.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.5625,
+      "rewards/margins": 22.25,
+      "rewards/rejected": -18.75,
+      "step": 2270
+    },
+    {
+      "epoch": 1.8269230769230769,
+      "grad_norm": 0.0001453323384470551,
+      "learning_rate": 2.1727515583259127e-07,
+      "logits/chosen": 1.0390625,
+      "logits/rejected": 1.3671875,
+      "logps/chosen": -159.0,
+      "logps/rejected": -316.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.40625,
+      "rewards/margins": 20.75,
+      "rewards/rejected": -17.375,
+      "step": 2280
+    },
+    {
+      "epoch": 1.8349358974358974,
+      "grad_norm": 4.835283260495599e-06,
+      "learning_rate": 2.1579103591570196e-07,
+      "logits/chosen": 0.8359375,
+      "logits/rejected": 1.375,
+      "logps/chosen": -153.0,
+      "logps/rejected": -314.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.8125,
+      "rewards/margins": 22.125,
+      "rewards/rejected": -18.25,
+      "step": 2290
+    },
+    {
+      "epoch": 1.842948717948718,
+      "grad_norm": 9.787977293423134e-05,
+      "learning_rate": 2.143069159988127e-07,
+      "logits/chosen": 1.0546875,
+      "logits/rejected": 1.3671875,
+      "logps/chosen": -180.0,
+      "logps/rejected": -320.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.59375,
+      "rewards/margins": 21.375,
+      "rewards/rejected": -17.75,
+      "step": 2300
+    },
+    {
+      "epoch": 1.8509615384615383,
+      "grad_norm": 0.003511562539920723,
+      "learning_rate": 2.128227960819234e-07,
+      "logits/chosen": 0.8515625,
+      "logits/rejected": 1.2890625,
+      "logps/chosen": -156.0,
+      "logps/rejected": -334.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.21875,
+      "rewards/margins": 23.5,
+      "rewards/rejected": -20.25,
+      "step": 2310
+    },
+    {
+      "epoch": 1.858974358974359,
+      "grad_norm": 0.0017406830607479289,
+      "learning_rate": 2.1133867616503413e-07,
+      "logits/chosen": 0.84375,
+      "logits/rejected": 1.4375,
+      "logps/chosen": -135.0,
+      "logps/rejected": -326.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.4375,
+      "rewards/margins": 22.875,
+      "rewards/rejected": -19.375,
+      "step": 2320
+    },
+    {
+      "epoch": 1.8669871794871795,
+      "grad_norm": 1.757377642753892e-05,
+      "learning_rate": 2.0985455624814487e-07,
+      "logits/chosen": 0.66015625,
+      "logits/rejected": 0.60546875,
+      "logps/chosen": -167.0,
+      "logps/rejected": -296.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.734375,
+      "rewards/margins": 21.125,
+      "rewards/rejected": -17.375,
+      "step": 2330
+    },
+    {
+      "epoch": 1.875,
+      "grad_norm": 9.585856156991564e-05,
+      "learning_rate": 2.0837043633125555e-07,
+      "logits/chosen": 0.7265625,
+      "logits/rejected": 1.0859375,
+      "logps/chosen": -189.0,
+      "logps/rejected": -318.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.71875,
+      "rewards/margins": 21.875,
+      "rewards/rejected": -18.125,
+      "step": 2340
+    },
+    {
+      "epoch": 1.8830128205128205,
+      "grad_norm": 2.717371131495447e-06,
+      "learning_rate": 2.0688631641436627e-07,
+      "logits/chosen": 0.68359375,
+      "logits/rejected": 0.91015625,
+      "logps/chosen": -173.0,
+      "logps/rejected": -310.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.953125,
+      "rewards/margins": 21.875,
+      "rewards/rejected": -17.875,
+      "step": 2350
+    },
+    {
+      "epoch": 1.891025641025641,
+      "grad_norm": 2.557631865211708e-07,
+      "learning_rate": 2.0540219649747698e-07,
+      "logits/chosen": 0.9140625,
+      "logits/rejected": 1.53125,
+      "logps/chosen": -175.0,
+      "logps/rejected": -328.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.8125,
+      "rewards/margins": 23.25,
+      "rewards/rejected": -19.375,
+      "step": 2360
+    },
+    {
+      "epoch": 1.8990384615384617,
+      "grad_norm": 8.115167511795904e-05,
+      "learning_rate": 2.0391807658058772e-07,
+      "logits/chosen": 0.91015625,
+      "logits/rejected": 1.2109375,
+      "logps/chosen": -186.0,
+      "logps/rejected": -312.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.953125,
+      "rewards/margins": 22.125,
+      "rewards/rejected": -18.25,
+      "step": 2370
+    },
+    {
+      "epoch": 1.907051282051282,
+      "grad_norm": 5.5980367488511575e-06,
+      "learning_rate": 2.024339566636984e-07,
+      "logits/chosen": 0.52734375,
+      "logits/rejected": 1.0859375,
+      "logps/chosen": -178.0,
+      "logps/rejected": -316.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.703125,
+      "rewards/margins": 22.375,
+      "rewards/rejected": -18.75,
+      "step": 2380
+    },
+    {
+      "epoch": 1.9150641025641026,
+      "grad_norm": 0.0025820105156682694,
+      "learning_rate": 2.0094983674680912e-07,
+      "logits/chosen": 0.8828125,
+      "logits/rejected": 1.3359375,
+      "logps/chosen": -150.0,
+      "logps/rejected": -300.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.34375,
+      "rewards/margins": 20.875,
+      "rewards/rejected": -17.5,
+      "step": 2390
+    },
+    {
+      "epoch": 1.9230769230769231,
+      "grad_norm": 7.136824036042445e-07,
+      "learning_rate": 1.9946571682991986e-07,
+      "logits/chosen": 0.78125,
+      "logits/rejected": 1.140625,
+      "logps/chosen": -149.0,
+      "logps/rejected": -328.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.890625,
+      "rewards/margins": 22.5,
+      "rewards/rejected": -19.625,
+      "step": 2400
+    },
+    {
+      "epoch": 1.9310897435897436,
+      "grad_norm": 0.0003371284846547073,
+      "learning_rate": 1.9798159691303057e-07,
+      "logits/chosen": 0.7890625,
+      "logits/rejected": 1.0078125,
+      "logps/chosen": -182.0,
+      "logps/rejected": -332.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.953125,
+      "rewards/margins": 23.0,
+      "rewards/rejected": -20.125,
+      "step": 2410
+    },
+    {
+      "epoch": 1.939102564102564,
+      "grad_norm": 7.032612597839408e-05,
+      "learning_rate": 1.964974769961413e-07,
+      "logits/chosen": 0.8125,
+      "logits/rejected": 1.0234375,
+      "logps/chosen": -155.0,
+      "logps/rejected": -330.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 2.890625,
+      "rewards/margins": 22.5,
+      "rewards/rejected": -19.625,
+      "step": 2420
+    },
+    {
+      "epoch": 1.9471153846153846,
+      "grad_norm": 0.00020818829472435283,
+      "learning_rate": 1.9501335707925197e-07,
+      "logits/chosen": 0.87890625,
+      "logits/rejected": 1.171875,
+      "logps/chosen": -192.0,
+      "logps/rejected": -338.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.546875,
+      "rewards/margins": 23.125,
+      "rewards/rejected": -19.625,
+      "step": 2430
+    },
+    {
+      "epoch": 1.9551282051282053,
+      "grad_norm": 8.466910776291965e-07,
+      "learning_rate": 1.9352923716236271e-07,
+      "logits/chosen": 0.7421875,
+      "logits/rejected": 1.375,
+      "logps/chosen": -212.0,
+      "logps/rejected": -330.0,
+      "loss": 0.0001,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.5,
+      "rewards/margins": 22.125,
+      "rewards/rejected": -18.625,
+      "step": 2440
+    },
+    {
+      "epoch": 1.9631410256410255,
+      "grad_norm": 3.7142533186252094e-06,
+      "learning_rate": 1.9204511724547343e-07,
+      "logits/chosen": 0.72265625,
+      "logits/rejected": 1.265625,
+      "logps/chosen": -172.0,
+      "logps/rejected": -348.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.875,
+      "rewards/margins": 23.625,
+      "rewards/rejected": -19.75,
+      "step": 2450
+    },
+    {
+      "epoch": 1.9711538461538463,
+      "grad_norm": 1.4211106326699326e-06,
+      "learning_rate": 1.9056099732858414e-07,
+      "logits/chosen": 0.9140625,
+      "logits/rejected": 1.2421875,
+      "logps/chosen": -160.0,
+      "logps/rejected": -332.0,
+      "loss": 0.0001,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.84375,
+      "rewards/margins": 24.0,
+      "rewards/rejected": -20.125,
+      "step": 2460
+    },
+    {
+      "epoch": 1.9791666666666665,
+      "grad_norm": 7.012330169691276e-07,
+      "learning_rate": 1.8907687741169488e-07,
+      "logits/chosen": 0.52734375,
+      "logits/rejected": 1.1640625,
+      "logps/chosen": -200.0,
+      "logps/rejected": -336.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.4375,
+      "rewards/margins": 22.0,
+      "rewards/rejected": -18.5,
+      "step": 2470
+    },
+    {
+      "epoch": 1.9871794871794872,
+      "grad_norm": 0.00015207925609359054,
+      "learning_rate": 1.8759275749480557e-07,
+      "logits/chosen": 0.921875,
+      "logits/rejected": 1.421875,
+      "logps/chosen": -203.0,
+      "logps/rejected": -328.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.296875,
+      "rewards/margins": 22.5,
+      "rewards/rejected": -19.25,
+      "step": 2480
+    },
+    {
+      "epoch": 1.9951923076923077,
+      "grad_norm": 1.0755143190145868e-06,
+      "learning_rate": 1.8610863757791628e-07,
+      "logits/chosen": 0.84375,
+      "logits/rejected": 1.3515625,
+      "logps/chosen": -168.0,
+      "logps/rejected": -332.0,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 3.75,
+      "rewards/margins": 23.75,
+      "rewards/rejected": -20.0,
+      "step": 2490
+    },
+    {
+      "epoch": 2.0,
+      "eval_logits/chosen": 0.84375,
+      "eval_logits/rejected": 1.3984375,
+      "eval_logps/chosen": -169.0,
+      "eval_logps/rejected": -334.0,
+      "eval_loss": 1.841681842051912e-05,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": 3.4375,
+      "eval_rewards/margins": 22.5,
+      "eval_rewards/rejected": -19.125,
+      "eval_runtime": 25.8897,
+      "eval_samples_per_second": 7.686,
+      "eval_steps_per_second": 0.966,
+      "step": 2496
     }
   ],
   "logging_steps": 10,