diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,17066 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.999297541394882,
+  "eval_steps": 400,
+  "global_step": 5604,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.002676032781401572,
+      "grad_norm": 11.496960384029425,
+      "learning_rate": 8.9126559714795e-09,
+      "logits/chosen": -0.07004348933696747,
+      "logits/rejected": 0.13604964315891266,
+      "logps/chosen": -1.716159462928772,
+      "logps/rejected": -1.8895246982574463,
+      "loss": 1.9598,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.716159462928772,
+      "rewards/margins": 0.17336519062519073,
+      "rewards/rejected": -1.8895246982574463,
+      "step": 5
+    },
+    {
+      "epoch": 0.005352065562803144,
+      "grad_norm": 24.62799600093432,
+      "learning_rate": 1.7825311942959e-08,
+      "logits/chosen": 0.011140784248709679,
+      "logits/rejected": 0.13161656260490417,
+      "logps/chosen": -1.8020946979522705,
+      "logps/rejected": -1.8448978662490845,
+      "loss": 2.0522,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.8020946979522705,
+      "rewards/margins": 0.042803239077329636,
+      "rewards/rejected": -1.8448978662490845,
+      "step": 10
+    },
+    {
+      "epoch": 0.008028098344204716,
+      "grad_norm": 22.60780341155583,
+      "learning_rate": 2.67379679144385e-08,
+      "logits/chosen": -0.025929506868124008,
+      "logits/rejected": 0.07177695631980896,
+      "logps/chosen": -1.6351503133773804,
+      "logps/rejected": -1.7645927667617798,
+      "loss": 1.9222,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -1.6351503133773804,
+      "rewards/margins": 0.1294424831867218,
+      "rewards/rejected": -1.7645927667617798,
+      "step": 15
+    },
+    {
+      "epoch": 0.010704131125606288,
+      "grad_norm": 12.75852372874483,
+      "learning_rate": 3.5650623885918e-08,
+      "logits/chosen": -0.031114911660552025,
+      "logits/rejected": 0.05350674316287041,
+      "logps/chosen": -1.7255808115005493,
+      "logps/rejected": -1.8063666820526123,
+      "loss": 2.0038,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -1.7255808115005493,
+      "rewards/margins": 0.08078588545322418,
+      "rewards/rejected": -1.8063666820526123,
+      "step": 20
+    },
+    {
+      "epoch": 0.013380163907007862,
+      "grad_norm": 30.50287118337272,
+      "learning_rate": 4.45632798573975e-08,
+      "logits/chosen": -0.0557979941368103,
+      "logits/rejected": 0.02790733054280281,
+      "logps/chosen": -1.8691844940185547,
+      "logps/rejected": -1.7790381908416748,
+      "loss": 2.176,
+      "rewards/accuracies": 0.38749998807907104,
+      "rewards/chosen": -1.8691844940185547,
+      "rewards/margins": -0.09014609456062317,
+      "rewards/rejected": -1.7790381908416748,
+      "step": 25
+    },
+    {
+      "epoch": 0.016056196688409432,
+      "grad_norm": 27.251095547414117,
+      "learning_rate": 5.3475935828877e-08,
+      "logits/chosen": -0.09042690694332123,
+      "logits/rejected": 0.001267892075702548,
+      "logps/chosen": -1.9062639474868774,
+      "logps/rejected": -1.830116868019104,
+      "loss": 2.1541,
+      "rewards/accuracies": 0.4437499940395355,
+      "rewards/chosen": -1.9062639474868774,
+      "rewards/margins": -0.07614694535732269,
+      "rewards/rejected": -1.830116868019104,
+      "step": 30
+    },
+    {
+      "epoch": 0.018732229469811006,
+      "grad_norm": 19.649059504433637,
+      "learning_rate": 6.23885918003565e-08,
+      "logits/chosen": -0.038590312004089355,
+      "logits/rejected": 0.12247200310230255,
+      "logps/chosen": -1.8448536396026611,
+      "logps/rejected": -1.9959800243377686,
+      "loss": 2.1077,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -1.8448536396026611,
+      "rewards/margins": 0.15112656354904175,
+      "rewards/rejected": -1.9959800243377686,
+      "step": 35
+    },
+    {
+      "epoch": 0.021408262251212576,
+      "grad_norm": 25.97747145346806,
+      "learning_rate": 7.1301247771836e-08,
+      "logits/chosen": 0.046131886541843414,
+      "logits/rejected": 0.22134876251220703,
+      "logps/chosen": -1.8790369033813477,
+      "logps/rejected": -1.7409080266952515,
+      "loss": 2.1579,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -1.8790369033813477,
+      "rewards/margins": -0.13812878727912903,
+      "rewards/rejected": -1.7409080266952515,
+      "step": 40
+    },
+    {
+      "epoch": 0.02408429503261415,
+      "grad_norm": 23.426944055276945,
+      "learning_rate": 8.021390374331551e-08,
+      "logits/chosen": 0.03349178284406662,
+      "logits/rejected": 0.23275916278362274,
+      "logps/chosen": -1.8335708379745483,
+      "logps/rejected": -1.8679958581924438,
+      "loss": 2.1043,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -1.8335708379745483,
+      "rewards/margins": 0.034425050020217896,
+      "rewards/rejected": -1.8679958581924438,
+      "step": 45
+    },
+    {
+      "epoch": 0.026760327814015723,
+      "grad_norm": 28.041207238046148,
+      "learning_rate": 8.9126559714795e-08,
+      "logits/chosen": -0.04500294104218483,
+      "logits/rejected": 0.10422907024621964,
+      "logps/chosen": -1.8920742273330688,
+      "logps/rejected": -1.7743374109268188,
+      "loss": 2.1584,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.8920742273330688,
+      "rewards/margins": -0.1177368313074112,
+      "rewards/rejected": -1.7743374109268188,
+      "step": 50
+    },
+    {
+      "epoch": 0.029436360595417294,
+      "grad_norm": 22.230131366220668,
+      "learning_rate": 9.80392156862745e-08,
+      "logits/chosen": -0.11886776983737946,
+      "logits/rejected": 0.09651211649179459,
+      "logps/chosen": -1.8243519067764282,
+      "logps/rejected": -1.8590061664581299,
+      "loss": 2.0787,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.8243519067764282,
+      "rewards/margins": 0.034654535353183746,
+      "rewards/rejected": -1.8590061664581299,
+      "step": 55
+    },
+    {
+      "epoch": 0.032112393376818864,
+      "grad_norm": 25.41905736053761,
+      "learning_rate": 1.06951871657754e-07,
+      "logits/chosen": -0.06625660508871078,
+      "logits/rejected": 0.1267094761133194,
+      "logps/chosen": -1.7776578664779663,
+      "logps/rejected": -1.8829160928726196,
+      "loss": 2.0253,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.7776578664779663,
+      "rewards/margins": 0.10525822639465332,
+      "rewards/rejected": -1.8829160928726196,
+      "step": 60
+    },
+    {
+      "epoch": 0.03478842615822044,
+      "grad_norm": 24.019383898594057,
+      "learning_rate": 1.158645276292335e-07,
+      "logits/chosen": -0.026983071118593216,
+      "logits/rejected": 0.11884398758411407,
+      "logps/chosen": -1.628180742263794,
+      "logps/rejected": -1.756899118423462,
+      "loss": 1.8999,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.628180742263794,
+      "rewards/margins": 0.12871839106082916,
+      "rewards/rejected": -1.756899118423462,
+      "step": 65
+    },
+    {
+      "epoch": 0.03746445893962201,
+      "grad_norm": 24.8180689671467,
+      "learning_rate": 1.24777183600713e-07,
+      "logits/chosen": -0.06905418634414673,
+      "logits/rejected": 0.08335807174444199,
+      "logps/chosen": -1.7545757293701172,
+      "logps/rejected": -1.7981780767440796,
+      "loss": 2.0317,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": -1.7545757293701172,
+      "rewards/margins": 0.043602414429187775,
+      "rewards/rejected": -1.7981780767440796,
+      "step": 70
+    },
+    {
+      "epoch": 0.04014049172102358,
+      "grad_norm": 20.509875985552213,
+      "learning_rate": 1.3368983957219251e-07,
+      "logits/chosen": -0.040019623935222626,
+      "logits/rejected": 0.1444050371646881,
+      "logps/chosen": -1.7451117038726807,
+      "logps/rejected": -2.000904083251953,
+      "loss": 1.9932,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.7451117038726807,
+      "rewards/margins": 0.25579261779785156,
+      "rewards/rejected": -2.000904083251953,
+      "step": 75
+    },
+    {
+      "epoch": 0.04281652450242515,
+      "grad_norm": 19.0701022504621,
+      "learning_rate": 1.42602495543672e-07,
+      "logits/chosen": -0.011737307533621788,
+      "logits/rejected": 0.09179870784282684,
+      "logps/chosen": -1.6793285608291626,
+      "logps/rejected": -1.7125403881072998,
+      "loss": 1.9591,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.6793285608291626,
+      "rewards/margins": 0.03321211040019989,
+      "rewards/rejected": -1.7125403881072998,
+      "step": 80
+    },
+    {
+      "epoch": 0.04549255728382673,
+      "grad_norm": 14.351006445287252,
+      "learning_rate": 1.5151515151515152e-07,
+      "logits/chosen": -0.16043472290039062,
+      "logits/rejected": 0.08176468312740326,
+      "logps/chosen": -1.74630868434906,
+      "logps/rejected": -1.9133342504501343,
+      "loss": 2.0241,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.74630868434906,
+      "rewards/margins": 0.16702556610107422,
+      "rewards/rejected": -1.9133342504501343,
+      "step": 85
+    },
+    {
+      "epoch": 0.0481685900652283,
+      "grad_norm": 25.696281391928444,
+      "learning_rate": 1.6042780748663102e-07,
+      "logits/chosen": 0.053802113980054855,
+      "logits/rejected": 0.014244931749999523,
+      "logps/chosen": -1.692808747291565,
+      "logps/rejected": -1.732717752456665,
+      "loss": 1.9809,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -1.692808747291565,
+      "rewards/margins": 0.03990894928574562,
+      "rewards/rejected": -1.732717752456665,
+      "step": 90
+    },
+    {
+      "epoch": 0.05084462284662987,
+      "grad_norm": 29.848111715590246,
+      "learning_rate": 1.693404634581105e-07,
+      "logits/chosen": -0.10460035502910614,
+      "logits/rejected": 0.04046844691038132,
+      "logps/chosen": -1.7314746379852295,
+      "logps/rejected": -1.857269287109375,
+      "loss": 1.9942,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.7314746379852295,
+      "rewards/margins": 0.12579476833343506,
+      "rewards/rejected": -1.857269287109375,
+      "step": 95
+    },
+    {
+      "epoch": 0.05352065562803145,
+      "grad_norm": 14.152630130218263,
+      "learning_rate": 1.7825311942959e-07,
+      "logits/chosen": -0.03871261700987816,
+      "logits/rejected": 0.02377936616539955,
+      "logps/chosen": -1.6145925521850586,
+      "logps/rejected": -1.7207527160644531,
+      "loss": 1.8863,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.6145925521850586,
+      "rewards/margins": 0.10616017878055573,
+      "rewards/rejected": -1.7207527160644531,
+      "step": 100
+    },
+    {
+      "epoch": 0.05619668840943302,
+      "grad_norm": 20.414035787261877,
+      "learning_rate": 1.8716577540106952e-07,
+      "logits/chosen": 0.03783528506755829,
+      "logits/rejected": 0.06228012964129448,
+      "logps/chosen": -1.533184289932251,
+      "logps/rejected": -1.7013075351715088,
+      "loss": 1.813,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.533184289932251,
+      "rewards/margins": 0.16812297701835632,
+      "rewards/rejected": -1.7013075351715088,
+      "step": 105
+    },
+    {
+      "epoch": 0.05887272119083459,
+      "grad_norm": 18.838681459332435,
+      "learning_rate": 1.96078431372549e-07,
+      "logits/chosen": -0.01743622124195099,
+      "logits/rejected": 0.07553930580615997,
+      "logps/chosen": -1.5218614339828491,
+      "logps/rejected": -1.581130027770996,
+      "loss": 1.8293,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.5218614339828491,
+      "rewards/margins": 0.05926854535937309,
+      "rewards/rejected": -1.581130027770996,
+      "step": 110
+    },
+    {
+      "epoch": 0.06154875397223616,
+      "grad_norm": 20.681911095989836,
+      "learning_rate": 2.049910873440285e-07,
+      "logits/chosen": 0.02094849944114685,
+      "logits/rejected": 0.22674188017845154,
+      "logps/chosen": -1.5216385126113892,
+      "logps/rejected": -1.7595542669296265,
+      "loss": 1.7828,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.5216385126113892,
+      "rewards/margins": 0.23791582882404327,
+      "rewards/rejected": -1.7595542669296265,
+      "step": 115
+    },
+    {
+      "epoch": 0.06422478675363773,
+      "grad_norm": 22.590931606448105,
+      "learning_rate": 2.13903743315508e-07,
+      "logits/chosen": -0.08596738427877426,
+      "logits/rejected": 0.08397196233272552,
+      "logps/chosen": -1.5576399564743042,
+      "logps/rejected": -1.660080909729004,
+      "loss": 1.8551,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.5576399564743042,
+      "rewards/margins": 0.10244102776050568,
+      "rewards/rejected": -1.660080909729004,
+      "step": 120
+    },
+    {
+      "epoch": 0.0669008195350393,
+      "grad_norm": 7.514452992243219,
+      "learning_rate": 2.2281639928698751e-07,
+      "logits/chosen": -0.10349993407726288,
+      "logits/rejected": 0.025051873177289963,
+      "logps/chosen": -1.509615182876587,
+      "logps/rejected": -1.4850200414657593,
+      "loss": 1.8367,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -1.509615182876587,
+      "rewards/margins": -0.02459501102566719,
+      "rewards/rejected": -1.4850200414657593,
+      "step": 125
+    },
+    {
+      "epoch": 0.06957685231644088,
+      "grad_norm": 26.418915644433355,
+      "learning_rate": 2.31729055258467e-07,
+      "logits/chosen": 0.009734541177749634,
+      "logits/rejected": 0.1401519626379013,
+      "logps/chosen": -1.5340511798858643,
+      "logps/rejected": -1.6470483541488647,
+      "loss": 1.8246,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -1.5340511798858643,
+      "rewards/margins": 0.11299731582403183,
+      "rewards/rejected": -1.6470483541488647,
+      "step": 130
+    },
+    {
+      "epoch": 0.07225288509784245,
+      "grad_norm": 22.937112750822322,
+      "learning_rate": 2.406417112299465e-07,
+      "logits/chosen": -0.0796973779797554,
+      "logits/rejected": 0.03641734644770622,
+      "logps/chosen": -1.5494601726531982,
+      "logps/rejected": -1.596419095993042,
+      "loss": 1.8564,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.5494601726531982,
+      "rewards/margins": 0.04695894196629524,
+      "rewards/rejected": -1.596419095993042,
+      "step": 135
+    },
+    {
+      "epoch": 0.07492891787924402,
+      "grad_norm": 14.481886404114174,
+      "learning_rate": 2.49554367201426e-07,
+      "logits/chosen": -0.08387432992458344,
+      "logits/rejected": 0.07397343963384628,
+      "logps/chosen": -1.4759531021118164,
+      "logps/rejected": -1.5443933010101318,
+      "loss": 1.8147,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -1.4759531021118164,
+      "rewards/margins": 0.06844006478786469,
+      "rewards/rejected": -1.5443933010101318,
+      "step": 140
+    },
+    {
+      "epoch": 0.0776049506606456,
+      "grad_norm": 13.900630224506688,
+      "learning_rate": 2.5846702317290554e-07,
+      "logits/chosen": -0.06520480662584305,
+      "logits/rejected": 0.08101221174001694,
+      "logps/chosen": -1.3628003597259521,
+      "logps/rejected": -1.4583301544189453,
+      "loss": 1.7122,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -1.3628003597259521,
+      "rewards/margins": 0.09552977979183197,
+      "rewards/rejected": -1.4583301544189453,
+      "step": 145
+    },
+    {
+      "epoch": 0.08028098344204716,
+      "grad_norm": 14.456992031762525,
+      "learning_rate": 2.6737967914438503e-07,
+      "logits/chosen": -0.12199876457452774,
+      "logits/rejected": 0.02296736277639866,
+      "logps/chosen": -1.3004138469696045,
+      "logps/rejected": -1.3106777667999268,
+      "loss": 1.6961,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.3004138469696045,
+      "rewards/margins": 0.010263770818710327,
+      "rewards/rejected": -1.3106777667999268,
+      "step": 150
+    },
+    {
+      "epoch": 0.08295701622344874,
+      "grad_norm": 10.003665109023784,
+      "learning_rate": 2.762923351158645e-07,
+      "logits/chosen": -0.15272387862205505,
+      "logits/rejected": -0.10898196697235107,
+      "logps/chosen": -1.3114030361175537,
+      "logps/rejected": -1.4270459413528442,
+      "loss": 1.6758,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.3114030361175537,
+      "rewards/margins": 0.1156429648399353,
+      "rewards/rejected": -1.4270459413528442,
+      "step": 155
+    },
+    {
+      "epoch": 0.0856330490048503,
+      "grad_norm": 10.058938611024177,
+      "learning_rate": 2.85204991087344e-07,
+      "logits/chosen": -0.20483560860157013,
+      "logits/rejected": -0.07443522661924362,
+      "logps/chosen": -1.4005239009857178,
+      "logps/rejected": -1.3797595500946045,
+      "loss": 1.7876,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -1.4005239009857178,
+      "rewards/margins": -0.020764362066984177,
+      "rewards/rejected": -1.3797595500946045,
+      "step": 160
+    },
+    {
+      "epoch": 0.08830908178625188,
+      "grad_norm": 12.766881003620881,
+      "learning_rate": 2.941176470588235e-07,
+      "logits/chosen": -0.115422323346138,
+      "logits/rejected": 0.043187130242586136,
+      "logps/chosen": -1.3161473274230957,
+      "logps/rejected": -1.3928664922714233,
+      "loss": 1.7146,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -1.3161473274230957,
+      "rewards/margins": 0.07671914994716644,
+      "rewards/rejected": -1.3928664922714233,
+      "step": 165
+    },
+    {
+      "epoch": 0.09098511456765346,
+      "grad_norm": 12.889165264352723,
+      "learning_rate": 3.0303030303030305e-07,
+      "logits/chosen": -0.1455298364162445,
+      "logits/rejected": -0.09741321206092834,
+      "logps/chosen": -1.4335427284240723,
+      "logps/rejected": -1.5023107528686523,
+      "loss": 1.7765,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.4335427284240723,
+      "rewards/margins": 0.0687679797410965,
+      "rewards/rejected": -1.5023107528686523,
+      "step": 170
+    },
+    {
+      "epoch": 0.09366114734905502,
+      "grad_norm": 10.673475631603585,
+      "learning_rate": 3.1194295900178254e-07,
+      "logits/chosen": -0.017373189330101013,
+      "logits/rejected": -0.023483578115701675,
+      "logps/chosen": -1.32071053981781,
+      "logps/rejected": -1.407671570777893,
+      "loss": 1.7025,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.32071053981781,
+      "rewards/margins": 0.08696107566356659,
+      "rewards/rejected": -1.407671570777893,
+      "step": 175
+    },
+    {
+      "epoch": 0.0963371801304566,
+      "grad_norm": 8.027679837174334,
+      "learning_rate": 3.2085561497326203e-07,
+      "logits/chosen": -0.05327599123120308,
+      "logits/rejected": -0.05693614482879639,
+      "logps/chosen": -1.3389848470687866,
+      "logps/rejected": -1.5398029088974,
+      "loss": 1.6771,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.3389848470687866,
+      "rewards/margins": 0.20081813633441925,
+      "rewards/rejected": -1.5398029088974,
+      "step": 180
+    },
+    {
+      "epoch": 0.09901321291185818,
+      "grad_norm": 8.508747965236294,
+      "learning_rate": 3.297682709447415e-07,
+      "logits/chosen": -0.18951360881328583,
+      "logits/rejected": -0.11156556755304337,
+      "logps/chosen": -1.331371545791626,
+      "logps/rejected": -1.378575325012207,
+      "loss": 1.7309,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -1.331371545791626,
+      "rewards/margins": 0.047203779220581055,
+      "rewards/rejected": -1.378575325012207,
+      "step": 185
+    },
+    {
+      "epoch": 0.10168924569325974,
+      "grad_norm": 10.290163511178912,
+      "learning_rate": 3.38680926916221e-07,
+      "logits/chosen": -0.1001301184296608,
+      "logits/rejected": 0.008160894736647606,
+      "logps/chosen": -1.2569787502288818,
+      "logps/rejected": -1.3878567218780518,
+      "loss": 1.6555,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.2569787502288818,
+      "rewards/margins": 0.13087785243988037,
+      "rewards/rejected": -1.3878567218780518,
+      "step": 190
+    },
+    {
+      "epoch": 0.10436527847466132,
+      "grad_norm": 8.298194138873017,
+      "learning_rate": 3.475935828877005e-07,
+      "logits/chosen": -0.016883565112948418,
+      "logits/rejected": 0.1294752061367035,
+      "logps/chosen": -1.2395333051681519,
+      "logps/rejected": -1.4013464450836182,
+      "loss": 1.6175,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.2395333051681519,
+      "rewards/margins": 0.16181306540966034,
+      "rewards/rejected": -1.4013464450836182,
+      "step": 195
+    },
+    {
+      "epoch": 0.1070413112560629,
+      "grad_norm": 21.57478799062816,
+      "learning_rate": 3.5650623885918e-07,
+      "logits/chosen": -0.10271809995174408,
+      "logits/rejected": 0.027957985177636147,
+      "logps/chosen": -1.3616082668304443,
+      "logps/rejected": -1.4011781215667725,
+      "loss": 1.7377,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.3616082668304443,
+      "rewards/margins": 0.03956984728574753,
+      "rewards/rejected": -1.4011781215667725,
+      "step": 200
+    },
+    {
+      "epoch": 0.10971734403746446,
+      "grad_norm": 15.11761693118118,
+      "learning_rate": 3.654188948306595e-07,
+      "logits/chosen": -0.10172367095947266,
+      "logits/rejected": 0.028435533866286278,
+      "logps/chosen": -1.272033452987671,
+      "logps/rejected": -1.3431034088134766,
+      "loss": 1.6765,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.272033452987671,
+      "rewards/margins": 0.07106985151767731,
+      "rewards/rejected": -1.3431034088134766,
+      "step": 205
+    },
+    {
+      "epoch": 0.11239337681886603,
+      "grad_norm": 12.162560812099604,
+      "learning_rate": 3.7433155080213904e-07,
+      "logits/chosen": -0.17484010756015778,
+      "logits/rejected": 0.00014435648336075246,
+      "logps/chosen": -1.3493441343307495,
+      "logps/rejected": -1.457792043685913,
+      "loss": 1.7159,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.3493441343307495,
+      "rewards/margins": 0.10844788700342178,
+      "rewards/rejected": -1.457792043685913,
+      "step": 210
+    },
+    {
+      "epoch": 0.1150694096002676,
+      "grad_norm": 10.505960016810631,
+      "learning_rate": 3.8324420677361853e-07,
+      "logits/chosen": -0.21476414799690247,
+      "logits/rejected": 0.015725497156381607,
+      "logps/chosen": -1.3726469278335571,
+      "logps/rejected": -1.4256699085235596,
+      "loss": 1.7294,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.3726469278335571,
+      "rewards/margins": 0.053022850304841995,
+      "rewards/rejected": -1.4256699085235596,
+      "step": 215
+    },
+    {
+      "epoch": 0.11774544238166917,
+      "grad_norm": 16.8831046999415,
+      "learning_rate": 3.92156862745098e-07,
+      "logits/chosen": 0.012892027385532856,
+      "logits/rejected": 0.10256718099117279,
+      "logps/chosen": -1.294647455215454,
+      "logps/rejected": -1.4366071224212646,
+      "loss": 1.657,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.294647455215454,
+      "rewards/margins": 0.14195974171161652,
+      "rewards/rejected": -1.4366071224212646,
+      "step": 220
+    },
+    {
+      "epoch": 0.12042147516307075,
+      "grad_norm": 7.742873673583737,
+      "learning_rate": 4.010695187165775e-07,
+      "logits/chosen": -0.11977468430995941,
+      "logits/rejected": 0.03806857019662857,
+      "logps/chosen": -1.3015865087509155,
+      "logps/rejected": -1.4278619289398193,
+      "loss": 1.6556,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.3015865087509155,
+      "rewards/margins": 0.12627527117729187,
+      "rewards/rejected": -1.4278619289398193,
+      "step": 225
+    },
+    {
+      "epoch": 0.12309750794447231,
+      "grad_norm": 8.81005179915772,
+      "learning_rate": 4.09982174688057e-07,
+      "logits/chosen": -0.0395035445690155,
+      "logits/rejected": 0.0292628463357687,
+      "logps/chosen": -1.3038604259490967,
+      "logps/rejected": -1.4596668481826782,
+      "loss": 1.6834,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.3038604259490967,
+      "rewards/margins": 0.1558062732219696,
+      "rewards/rejected": -1.4596668481826782,
+      "step": 230
+    },
+    {
+      "epoch": 0.1257735407258739,
+      "grad_norm": 12.371393950608745,
+      "learning_rate": 4.188948306595365e-07,
+      "logits/chosen": -0.0051063718274235725,
+      "logits/rejected": 0.1200011596083641,
+      "logps/chosen": -1.2733403444290161,
+      "logps/rejected": -1.43751859664917,
+      "loss": 1.6422,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.2733403444290161,
+      "rewards/margins": 0.16417834162712097,
+      "rewards/rejected": -1.43751859664917,
+      "step": 235
+    },
+    {
+      "epoch": 0.12844957350727546,
+      "grad_norm": 6.457958618778695,
+      "learning_rate": 4.27807486631016e-07,
+      "logits/chosen": -0.053750477731227875,
+      "logits/rejected": 0.06239823251962662,
+      "logps/chosen": -1.289251446723938,
+      "logps/rejected": -1.4710654020309448,
+      "loss": 1.6871,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.289251446723938,
+      "rewards/margins": 0.1818140149116516,
+      "rewards/rejected": -1.4710654020309448,
+      "step": 240
+    },
+    {
+      "epoch": 0.13112560628867703,
+      "grad_norm": 8.860937119366293,
+      "learning_rate": 4.3672014260249554e-07,
+      "logits/chosen": 4.385709689813666e-05,
+      "logits/rejected": 0.10791563987731934,
+      "logps/chosen": -1.4089033603668213,
+      "logps/rejected": -1.4336974620819092,
+      "loss": 1.775,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.4089033603668213,
+      "rewards/margins": 0.024794071912765503,
+      "rewards/rejected": -1.4336974620819092,
+      "step": 245
+    },
+    {
+      "epoch": 0.1338016390700786,
+      "grad_norm": 10.341642763213565,
+      "learning_rate": 4.4563279857397503e-07,
+      "logits/chosen": -0.06541736423969269,
+      "logits/rejected": 0.09154494106769562,
+      "logps/chosen": -1.2923893928527832,
+      "logps/rejected": -1.3493139743804932,
+      "loss": 1.701,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -1.2923893928527832,
+      "rewards/margins": 0.056924544274806976,
+      "rewards/rejected": -1.3493139743804932,
+      "step": 250
+    },
+    {
+      "epoch": 0.1364776718514802,
+      "grad_norm": 8.995445203244538,
+      "learning_rate": 4.545454545454545e-07,
+      "logits/chosen": -0.03822184354066849,
+      "logits/rejected": 0.09474680572748184,
+      "logps/chosen": -1.2578426599502563,
+      "logps/rejected": -1.3610799312591553,
+      "loss": 1.6498,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.2578426599502563,
+      "rewards/margins": 0.10323736816644669,
+      "rewards/rejected": -1.3610799312591553,
+      "step": 255
+    },
+    {
+      "epoch": 0.13915370463288176,
+      "grad_norm": 7.808941045370267,
+      "learning_rate": 4.63458110516934e-07,
+      "logits/chosen": -0.2500733733177185,
+      "logits/rejected": -0.1506270170211792,
+      "logps/chosen": -1.3432053327560425,
+      "logps/rejected": -1.4983842372894287,
+      "loss": 1.6675,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.3432053327560425,
+      "rewards/margins": 0.15517887473106384,
+      "rewards/rejected": -1.4983842372894287,
+      "step": 260
+    },
+    {
+      "epoch": 0.1418297374142833,
+      "grad_norm": 11.263217166611032,
+      "learning_rate": 4.723707664884135e-07,
+      "logits/chosen": -0.11429083347320557,
+      "logits/rejected": -0.03443082422018051,
+      "logps/chosen": -1.333311676979065,
+      "logps/rejected": -1.495581030845642,
+      "loss": 1.6814,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.333311676979065,
+      "rewards/margins": 0.16226927936077118,
+      "rewards/rejected": -1.495581030845642,
+      "step": 265
+    },
+    {
+      "epoch": 0.1445057701956849,
+      "grad_norm": 8.04536803191158,
+      "learning_rate": 4.81283422459893e-07,
+      "logits/chosen": -0.09445016086101532,
+      "logits/rejected": 0.0262304600328207,
+      "logps/chosen": -1.3086737394332886,
+      "logps/rejected": -1.4043707847595215,
+      "loss": 1.6979,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.3086737394332886,
+      "rewards/margins": 0.09569709002971649,
+      "rewards/rejected": -1.4043707847595215,
+      "step": 270
+    },
+    {
+      "epoch": 0.14718180297708647,
+      "grad_norm": 9.923872351321576,
+      "learning_rate": 4.901960784313725e-07,
+      "logits/chosen": -0.028187870979309082,
+      "logits/rejected": 0.06616386771202087,
+      "logps/chosen": -1.2604142427444458,
+      "logps/rejected": -1.4108034372329712,
+      "loss": 1.6666,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.2604142427444458,
+      "rewards/margins": 0.15038911998271942,
+      "rewards/rejected": -1.4108034372329712,
+      "step": 275
+    },
+    {
+      "epoch": 0.14985783575848804,
+      "grad_norm": 10.338563914076289,
+      "learning_rate": 4.99108734402852e-07,
+      "logits/chosen": -0.10596181452274323,
+      "logits/rejected": 0.043247222900390625,
+      "logps/chosen": -1.3092851638793945,
+      "logps/rejected": -1.4032866954803467,
+      "loss": 1.6684,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.3092851638793945,
+      "rewards/margins": 0.09400156885385513,
+      "rewards/rejected": -1.4032866954803467,
+      "step": 280
+    },
+    {
+      "epoch": 0.15253386853988962,
+      "grad_norm": 9.301649752575466,
+      "learning_rate": 5.080213903743315e-07,
+      "logits/chosen": -0.07537764310836792,
+      "logits/rejected": 0.05667581036686897,
+      "logps/chosen": -1.339205026626587,
+      "logps/rejected": -1.416961669921875,
+      "loss": 1.7063,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.339205026626587,
+      "rewards/margins": 0.07775678485631943,
+      "rewards/rejected": -1.416961669921875,
+      "step": 285
+    },
+    {
+      "epoch": 0.1552099013212912,
+      "grad_norm": 8.371480037795749,
+      "learning_rate": 5.169340463458111e-07,
+      "logits/chosen": -0.137905552983284,
+      "logits/rejected": 0.1398901641368866,
+      "logps/chosen": -1.3563731908798218,
+      "logps/rejected": -1.47064208984375,
+      "loss": 1.6818,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.3563731908798218,
+      "rewards/margins": 0.11426875740289688,
+      "rewards/rejected": -1.47064208984375,
+      "step": 290
+    },
+    {
+      "epoch": 0.15788593410269275,
+      "grad_norm": 12.564451448904155,
+      "learning_rate": 5.258467023172905e-07,
+      "logits/chosen": -0.08563229441642761,
+      "logits/rejected": -0.029086634516716003,
+      "logps/chosen": -1.2586605548858643,
+      "logps/rejected": -1.3940225839614868,
+      "loss": 1.6419,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.2586605548858643,
+      "rewards/margins": 0.13536228239536285,
+      "rewards/rejected": -1.3940225839614868,
+      "step": 295
+    },
+    {
+      "epoch": 0.16056196688409433,
+      "grad_norm": 8.503279331236856,
+      "learning_rate": 5.347593582887701e-07,
+      "logits/chosen": -0.08329950273036957,
+      "logits/rejected": 0.07354854047298431,
+      "logps/chosen": -1.2957587242126465,
+      "logps/rejected": -1.3708839416503906,
+      "loss": 1.6978,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.2957587242126465,
+      "rewards/margins": 0.07512523233890533,
+      "rewards/rejected": -1.3708839416503906,
+      "step": 300
+    },
+    {
+      "epoch": 0.1632379996654959,
+      "grad_norm": 7.082413964772183,
+      "learning_rate": 5.436720142602496e-07,
+      "logits/chosen": -0.031066909432411194,
+      "logits/rejected": 0.0350339449942112,
+      "logps/chosen": -1.3960840702056885,
+      "logps/rejected": -1.4059756994247437,
+      "loss": 1.7538,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.3960840702056885,
+      "rewards/margins": 0.009891606867313385,
+      "rewards/rejected": -1.4059756994247437,
+      "step": 305
+    },
+    {
+      "epoch": 0.16591403244689748,
+      "grad_norm": 8.740780332970166,
+      "learning_rate": 5.52584670231729e-07,
+      "logits/chosen": -0.2536133825778961,
+      "logits/rejected": -0.17383472621440887,
+      "logps/chosen": -1.362154483795166,
+      "logps/rejected": -1.4443097114562988,
+      "loss": 1.731,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.362154483795166,
+      "rewards/margins": 0.08215537667274475,
+      "rewards/rejected": -1.4443097114562988,
+      "step": 310
+    },
+    {
+      "epoch": 0.16859006522829906,
+      "grad_norm": 10.359867234690551,
+      "learning_rate": 5.614973262032086e-07,
+      "logits/chosen": -0.03551404923200607,
+      "logits/rejected": 0.10857124626636505,
+      "logps/chosen": -1.3527511358261108,
+      "logps/rejected": -1.4951927661895752,
+      "loss": 1.6965,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.3527511358261108,
+      "rewards/margins": 0.14244171977043152,
+      "rewards/rejected": -1.4951927661895752,
+      "step": 315
+    },
+    {
+      "epoch": 0.1712660980097006,
+      "grad_norm": 6.856055712472328,
+      "learning_rate": 5.70409982174688e-07,
+      "logits/chosen": -0.09569795429706573,
+      "logits/rejected": 0.026764288544654846,
+      "logps/chosen": -1.3127437829971313,
+      "logps/rejected": -1.3622338771820068,
+      "loss": 1.7075,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.3127437829971313,
+      "rewards/margins": 0.04949017986655235,
+      "rewards/rejected": -1.3622338771820068,
+      "step": 320
+    },
+    {
+      "epoch": 0.17394213079110218,
+      "grad_norm": 9.565706670007815,
+      "learning_rate": 5.793226381461676e-07,
+      "logits/chosen": -0.15836867690086365,
+      "logits/rejected": -0.05464999005198479,
+      "logps/chosen": -1.3131455183029175,
+      "logps/rejected": -1.5495985746383667,
+      "loss": 1.6645,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.3131455183029175,
+      "rewards/margins": 0.23645305633544922,
+      "rewards/rejected": -1.5495985746383667,
+      "step": 325
+    },
+    {
+      "epoch": 0.17661816357250376,
+      "grad_norm": 9.880050272421682,
+      "learning_rate": 5.88235294117647e-07,
+      "logits/chosen": -0.062169916927814484,
+      "logits/rejected": 0.06957022100687027,
+      "logps/chosen": -1.3260176181793213,
+      "logps/rejected": -1.4939022064208984,
+      "loss": 1.6917,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.3260176181793213,
+      "rewards/margins": 0.16788452863693237,
+      "rewards/rejected": -1.4939022064208984,
+      "step": 330
+    },
+    {
+      "epoch": 0.17929419635390534,
+      "grad_norm": 9.66922885238171,
+      "learning_rate": 5.971479500891266e-07,
+      "logits/chosen": 0.006079360842704773,
+      "logits/rejected": 0.09727548807859421,
+      "logps/chosen": -1.3392069339752197,
+      "logps/rejected": -1.3705189228057861,
+      "loss": 1.7279,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.3392069339752197,
+      "rewards/margins": 0.031312037259340286,
+      "rewards/rejected": -1.3705189228057861,
+      "step": 335
+    },
+    {
+      "epoch": 0.18197022913530692,
+      "grad_norm": 11.033412727973458,
+      "learning_rate": 6.060606060606061e-07,
+      "logits/chosen": -0.03210308775305748,
+      "logits/rejected": 0.10471953451633453,
+      "logps/chosen": -1.3952913284301758,
+      "logps/rejected": -1.4770443439483643,
+      "loss": 1.759,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.3952913284301758,
+      "rewards/margins": 0.08175303041934967,
+      "rewards/rejected": -1.4770443439483643,
+      "step": 340
+    },
+    {
+      "epoch": 0.1846462619167085,
+      "grad_norm": 10.896535078719086,
+      "learning_rate": 6.149732620320855e-07,
+      "logits/chosen": 0.03205911070108414,
+      "logits/rejected": 0.057826824486255646,
+      "logps/chosen": -1.3063201904296875,
+      "logps/rejected": -1.4427855014801025,
+      "loss": 1.6554,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.3063201904296875,
+      "rewards/margins": 0.13646534085273743,
+      "rewards/rejected": -1.4427855014801025,
+      "step": 345
+    },
+    {
+      "epoch": 0.18732229469811004,
+      "grad_norm": 10.28054144795725,
+      "learning_rate": 6.238859180035651e-07,
+      "logits/chosen": -0.010967472568154335,
+      "logits/rejected": 0.07389771938323975,
+      "logps/chosen": -1.2843072414398193,
+      "logps/rejected": -1.399267554283142,
+      "loss": 1.6701,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.2843072414398193,
+      "rewards/margins": 0.1149604544043541,
+      "rewards/rejected": -1.399267554283142,
+      "step": 350
+    },
+    {
+      "epoch": 0.18999832747951162,
+      "grad_norm": 11.039648213043902,
+      "learning_rate": 6.327985739750445e-07,
+      "logits/chosen": -0.10593543201684952,
+      "logits/rejected": 0.10453915596008301,
+      "logps/chosen": -1.3861037492752075,
+      "logps/rejected": -1.4115387201309204,
+      "loss": 1.7495,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -1.3861037492752075,
+      "rewards/margins": 0.025434961542487144,
+      "rewards/rejected": -1.4115387201309204,
+      "step": 355
+    },
+    {
+      "epoch": 0.1926743602609132,
+      "grad_norm": 9.815585300420476,
+      "learning_rate": 6.417112299465241e-07,
+      "logits/chosen": -0.07406492531299591,
+      "logits/rejected": 0.0021810054313391447,
+      "logps/chosen": -1.315483808517456,
+      "logps/rejected": -1.4323620796203613,
+      "loss": 1.7339,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.315483808517456,
+      "rewards/margins": 0.11687831580638885,
+      "rewards/rejected": -1.4323620796203613,
+      "step": 360
+    },
+    {
+      "epoch": 0.19535039304231477,
+      "grad_norm": 11.993286618551066,
+      "learning_rate": 6.506238859180035e-07,
+      "logits/chosen": 0.007736300118267536,
+      "logits/rejected": 0.08413462340831757,
+      "logps/chosen": -1.292368769645691,
+      "logps/rejected": -1.3933672904968262,
+      "loss": 1.6824,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.292368769645691,
+      "rewards/margins": 0.10099846124649048,
+      "rewards/rejected": -1.3933672904968262,
+      "step": 365
+    },
+    {
+      "epoch": 0.19802642582371635,
+      "grad_norm": 6.775568693955582,
+      "learning_rate": 6.59536541889483e-07,
+      "logits/chosen": -0.03933990001678467,
+      "logits/rejected": 0.043112464249134064,
+      "logps/chosen": -1.288225531578064,
+      "logps/rejected": -1.329222321510315,
+      "loss": 1.6894,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.288225531578064,
+      "rewards/margins": 0.040996819734573364,
+      "rewards/rejected": -1.329222321510315,
+      "step": 370
+    },
+    {
+      "epoch": 0.2007024586051179,
+      "grad_norm": 15.797955341210931,
+      "learning_rate": 6.684491978609626e-07,
+      "logits/chosen": -0.0702676773071289,
+      "logits/rejected": 0.08288853615522385,
+      "logps/chosen": -1.2703843116760254,
+      "logps/rejected": -1.4104673862457275,
+      "loss": 1.6667,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.2703843116760254,
+      "rewards/margins": 0.1400829255580902,
+      "rewards/rejected": -1.4104673862457275,
+      "step": 375
+    },
+    {
+      "epoch": 0.20337849138651948,
+      "grad_norm": 9.3923973841865,
+      "learning_rate": 6.77361853832442e-07,
+      "logits/chosen": -0.05306627228856087,
+      "logits/rejected": 0.02266976237297058,
+      "logps/chosen": -1.2857197523117065,
+      "logps/rejected": -1.4401063919067383,
+      "loss": 1.6482,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.2857197523117065,
+      "rewards/margins": 0.1543864607810974,
+      "rewards/rejected": -1.4401063919067383,
+      "step": 380
+    },
+    {
+      "epoch": 0.20605452416792105,
+      "grad_norm": 5.749278727008655,
+      "learning_rate": 6.862745098039216e-07,
+      "logits/chosen": 0.024824131280183792,
+      "logits/rejected": 0.10414397716522217,
+      "logps/chosen": -1.383319616317749,
+      "logps/rejected": -1.372910976409912,
+      "loss": 1.7442,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.383319616317749,
+      "rewards/margins": -0.010408599860966206,
+      "rewards/rejected": -1.372910976409912,
+      "step": 385
+    },
+    {
+      "epoch": 0.20873055694932263,
+      "grad_norm": 10.617790928113745,
+      "learning_rate": 6.95187165775401e-07,
+      "logits/chosen": 0.06647435575723648,
+      "logits/rejected": 0.22994470596313477,
+      "logps/chosen": -1.3744819164276123,
+      "logps/rejected": -1.4345366954803467,
+      "loss": 1.7584,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -1.3744819164276123,
+      "rewards/margins": 0.06005479767918587,
+      "rewards/rejected": -1.4345366954803467,
+      "step": 390
+    },
+    {
+      "epoch": 0.2114065897307242,
+      "grad_norm": 7.745401478461128,
+      "learning_rate": 7.040998217468806e-07,
+      "logits/chosen": -0.0460563488304615,
+      "logits/rejected": 0.10764249414205551,
+      "logps/chosen": -1.3257935047149658,
+      "logps/rejected": -1.3358434438705444,
+      "loss": 1.7054,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.3257935047149658,
+      "rewards/margins": 0.010050171986222267,
+      "rewards/rejected": -1.3358434438705444,
+      "step": 395
+    },
+    {
+      "epoch": 0.2140826225121258,
+      "grad_norm": 7.754185347608746,
+      "learning_rate": 7.1301247771836e-07,
+      "logits/chosen": 0.05485847592353821,
+      "logits/rejected": 0.14413060247898102,
+      "logps/chosen": -1.3048384189605713,
+      "logps/rejected": -1.3969600200653076,
+      "loss": 1.6555,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.3048384189605713,
+      "rewards/margins": 0.09212164580821991,
+      "rewards/rejected": -1.3969600200653076,
+      "step": 400
+    },
+    {
+      "epoch": 0.2140826225121258,
+      "eval_logits/chosen": 0.3137153089046478,
+      "eval_logits/rejected": 0.40302950143814087,
+      "eval_logps/chosen": -1.3383382558822632,
+      "eval_logps/rejected": -1.4639986753463745,
+      "eval_loss": 1.6940995454788208,
+      "eval_rewards/accuracies": 0.5556379556655884,
+      "eval_rewards/chosen": -1.3383382558822632,
+      "eval_rewards/margins": 0.12566043436527252,
+      "eval_rewards/rejected": -1.4639986753463745,
+      "eval_runtime": 41.5943,
+      "eval_samples_per_second": 32.336,
+      "eval_steps_per_second": 8.102,
+      "step": 400
+    },
+    {
+      "epoch": 0.21675865529352734,
+      "grad_norm": 8.754336696951281,
+      "learning_rate": 7.219251336898395e-07,
+      "logits/chosen": -0.005072087049484253,
+      "logits/rejected": 0.08546868711709976,
+      "logps/chosen": -1.3079750537872314,
+      "logps/rejected": -1.371268630027771,
+      "loss": 1.7065,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.3079750537872314,
+      "rewards/margins": 0.0632937103509903,
+      "rewards/rejected": -1.371268630027771,
+      "step": 405
+    },
+    {
+      "epoch": 0.2194346880749289,
+      "grad_norm": 10.519751722000656,
+      "learning_rate": 7.30837789661319e-07,
+      "logits/chosen": 0.0018633157014846802,
+      "logits/rejected": 0.12912698090076447,
+      "logps/chosen": -1.2822954654693604,
+      "logps/rejected": -1.3665344715118408,
+      "loss": 1.6734,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.2822954654693604,
+      "rewards/margins": 0.08423889428377151,
+      "rewards/rejected": -1.3665344715118408,
+      "step": 410
+    },
+    {
+      "epoch": 0.2221107208563305,
+      "grad_norm": 6.074341910212351,
+      "learning_rate": 7.397504456327985e-07,
+      "logits/chosen": 0.004594428930431604,
+      "logits/rejected": 0.03903172165155411,
+      "logps/chosen": -1.2817447185516357,
+      "logps/rejected": -1.4341824054718018,
+      "loss": 1.6418,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.2817447185516357,
+      "rewards/margins": 0.15243759751319885,
+      "rewards/rejected": -1.4341824054718018,
+      "step": 415
+    },
+    {
+      "epoch": 0.22478675363773207,
+      "grad_norm": 7.604611473074622,
+      "learning_rate": 7.486631016042781e-07,
+      "logits/chosen": -0.015866711735725403,
+      "logits/rejected": 0.16937878727912903,
+      "logps/chosen": -1.2663681507110596,
+      "logps/rejected": -1.3563402891159058,
+      "loss": 1.6679,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.2663681507110596,
+      "rewards/margins": 0.08997206389904022,
+      "rewards/rejected": -1.3563402891159058,
+      "step": 420
+    },
+    {
+      "epoch": 0.22746278641913364,
+      "grad_norm": 6.717082718541665,
+      "learning_rate": 7.575757575757575e-07,
+      "logits/chosen": -0.0675196424126625,
+      "logits/rejected": 0.12153605371713638,
+      "logps/chosen": -1.306672215461731,
+      "logps/rejected": -1.4570428133010864,
+      "loss": 1.6619,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.306672215461731,
+      "rewards/margins": 0.15037064254283905,
+      "rewards/rejected": -1.4570428133010864,
+      "step": 425
+    },
+    {
+      "epoch": 0.2301388192005352,
+      "grad_norm": 7.371899282836315,
+      "learning_rate": 7.664884135472371e-07,
+      "logits/chosen": -0.08296699821949005,
+      "logits/rejected": 0.10469052940607071,
+      "logps/chosen": -1.3389928340911865,
+      "logps/rejected": -1.4607927799224854,
+      "loss": 1.6907,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.3389928340911865,
+      "rewards/margins": 0.12179992347955704,
+      "rewards/rejected": -1.4607927799224854,
+      "step": 430
+    },
+    {
+      "epoch": 0.23281485198193677,
+      "grad_norm": 12.06561551172007,
+      "learning_rate": 7.754010695187165e-07,
+      "logits/chosen": -0.012138782069087029,
+      "logits/rejected": 0.07224924117326736,
+      "logps/chosen": -1.2074350118637085,
+      "logps/rejected": -1.3375585079193115,
+      "loss": 1.6144,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.2074350118637085,
+      "rewards/margins": 0.13012345135211945,
+      "rewards/rejected": -1.3375585079193115,
+      "step": 435
+    },
+    {
+      "epoch": 0.23549088476333835,
+      "grad_norm": 7.575184535702604,
+      "learning_rate": 7.84313725490196e-07,
+      "logits/chosen": -0.0038666813634335995,
+      "logits/rejected": 0.08154430240392685,
+      "logps/chosen": -1.2854907512664795,
+      "logps/rejected": -1.368265986442566,
+      "loss": 1.6762,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.2854907512664795,
+      "rewards/margins": 0.08277542889118195,
+      "rewards/rejected": -1.368265986442566,
+      "step": 440
+    },
+    {
+      "epoch": 0.23816691754473993,
+      "grad_norm": 8.138193319450046,
+      "learning_rate": 7.932263814616755e-07,
+      "logits/chosen": -0.061522066593170166,
+      "logits/rejected": 0.04313060641288757,
+      "logps/chosen": -1.3112833499908447,
+      "logps/rejected": -1.433841347694397,
+      "loss": 1.6972,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.3112833499908447,
+      "rewards/margins": 0.12255801260471344,
+      "rewards/rejected": -1.433841347694397,
+      "step": 445
+    },
+    {
+      "epoch": 0.2408429503261415,
+      "grad_norm": 8.640186349405695,
+      "learning_rate": 8.02139037433155e-07,
+      "logits/chosen": -0.020474979653954506,
+      "logits/rejected": 0.09682993590831757,
+      "logps/chosen": -1.2927807569503784,
+      "logps/rejected": -1.4289346933364868,
+      "loss": 1.6518,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.2927807569503784,
+      "rewards/margins": 0.13615401089191437,
+      "rewards/rejected": -1.4289346933364868,
+      "step": 450
+    },
+    {
+      "epoch": 0.24351898310754308,
+      "grad_norm": 9.540609815035786,
+      "learning_rate": 8.110516934046346e-07,
+      "logits/chosen": -0.01543851476162672,
+      "logits/rejected": 0.07130376249551773,
+      "logps/chosen": -1.2424728870391846,
+      "logps/rejected": -1.41982102394104,
+      "loss": 1.6114,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.2424728870391846,
+      "rewards/margins": 0.1773480623960495,
+      "rewards/rejected": -1.41982102394104,
+      "step": 455
+    },
+    {
+      "epoch": 0.24619501588894463,
+      "grad_norm": 8.326447292688538,
+      "learning_rate": 8.19964349376114e-07,
+      "logits/chosen": -0.16184619069099426,
+      "logits/rejected": -0.051874689757823944,
+      "logps/chosen": -1.3679503202438354,
+      "logps/rejected": -1.4119255542755127,
+      "loss": 1.7356,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -1.3679503202438354,
+      "rewards/margins": 0.0439753457903862,
+      "rewards/rejected": -1.4119255542755127,
+      "step": 460
+    },
+    {
+      "epoch": 0.2488710486703462,
+      "grad_norm": 9.473883720691502,
+      "learning_rate": 8.288770053475936e-07,
+      "logits/chosen": 0.11290111392736435,
+      "logits/rejected": 0.12598662078380585,
+      "logps/chosen": -1.2719142436981201,
+      "logps/rejected": -1.4320366382598877,
+      "loss": 1.6722,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.2719142436981201,
+      "rewards/margins": 0.160122349858284,
+      "rewards/rejected": -1.4320366382598877,
+      "step": 465
+    },
+    {
+      "epoch": 0.2515470814517478,
+      "grad_norm": 9.555764432813708,
+      "learning_rate": 8.37789661319073e-07,
+      "logits/chosen": 0.1309710443019867,
+      "logits/rejected": 0.07963626086711884,
+      "logps/chosen": -1.234974980354309,
+      "logps/rejected": -1.4127203226089478,
+      "loss": 1.6056,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.234974980354309,
+      "rewards/margins": 0.17774544656276703,
+      "rewards/rejected": -1.4127203226089478,
+      "step": 470
+    },
+    {
+      "epoch": 0.25422311423314936,
+      "grad_norm": 6.846149735650059,
+      "learning_rate": 8.467023172905525e-07,
+      "logits/chosen": -0.0759991854429245,
+      "logits/rejected": 0.0541856586933136,
+      "logps/chosen": -1.299948811531067,
+      "logps/rejected": -1.4895131587982178,
+      "loss": 1.6333,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.299948811531067,
+      "rewards/margins": 0.18956440687179565,
+      "rewards/rejected": -1.4895131587982178,
+      "step": 475
+    },
+    {
+      "epoch": 0.2568991470145509,
+      "grad_norm": 12.67935847973589,
+      "learning_rate": 8.55614973262032e-07,
+      "logits/chosen": -0.06495614349842072,
+      "logits/rejected": 0.12569716572761536,
+      "logps/chosen": -1.2725584506988525,
+      "logps/rejected": -1.3375532627105713,
+      "loss": 1.7014,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.2725584506988525,
+      "rewards/margins": 0.06499501317739487,
+      "rewards/rejected": -1.3375532627105713,
+      "step": 480
+    },
+    {
+      "epoch": 0.2595751797959525,
+      "grad_norm": 11.367772552764144,
+      "learning_rate": 8.645276292335115e-07,
+      "logits/chosen": -0.0036151937674731016,
+      "logits/rejected": 0.030773457139730453,
+      "logps/chosen": -1.3656898736953735,
+      "logps/rejected": -1.435654878616333,
+      "loss": 1.7304,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.3656898736953735,
+      "rewards/margins": 0.06996499001979828,
+      "rewards/rejected": -1.435654878616333,
+      "step": 485
+    },
+    {
+      "epoch": 0.26225121257735406,
+      "grad_norm": 8.665607148566286,
+      "learning_rate": 8.734402852049911e-07,
+      "logits/chosen": -0.01699291542172432,
+      "logits/rejected": 0.05099986121058464,
+      "logps/chosen": -1.302870750427246,
+      "logps/rejected": -1.3726005554199219,
+      "loss": 1.7032,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.302870750427246,
+      "rewards/margins": 0.06972992420196533,
+      "rewards/rejected": -1.3726005554199219,
+      "step": 490
+    },
+    {
+      "epoch": 0.26492724535875567,
+      "grad_norm": 9.49301727299634,
+      "learning_rate": 8.823529411764705e-07,
+      "logits/chosen": -0.05295737460255623,
+      "logits/rejected": -0.03520811349153519,
+      "logps/chosen": -1.3093057870864868,
+      "logps/rejected": -1.4187067747116089,
+      "loss": 1.6828,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.3093057870864868,
+      "rewards/margins": 0.10940104722976685,
+      "rewards/rejected": -1.4187067747116089,
+      "step": 495
+    },
+    {
+      "epoch": 0.2676032781401572,
+      "grad_norm": 8.428671515813905,
+      "learning_rate": 8.912655971479501e-07,
+      "logits/chosen": -0.04793981835246086,
+      "logits/rejected": 0.04994889721274376,
+      "logps/chosen": -1.2275326251983643,
+      "logps/rejected": -1.379403829574585,
+      "loss": 1.6185,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.2275326251983643,
+      "rewards/margins": 0.15187107026576996,
+      "rewards/rejected": -1.379403829574585,
+      "step": 500
+    },
+    {
+      "epoch": 0.27027931092155877,
+      "grad_norm": 7.9895540595139956,
+      "learning_rate": 9.001782531194295e-07,
+      "logits/chosen": -0.062299322336912155,
+      "logits/rejected": 0.07209259271621704,
+      "logps/chosen": -1.3436644077301025,
+      "logps/rejected": -1.377514362335205,
+      "loss": 1.6953,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.3436644077301025,
+      "rewards/margins": 0.03384983539581299,
+      "rewards/rejected": -1.377514362335205,
+      "step": 505
+    },
+    {
+      "epoch": 0.2729553437029604,
+      "grad_norm": 8.131935548042284,
+      "learning_rate": 9.09090909090909e-07,
+      "logits/chosen": 0.07910905033349991,
+      "logits/rejected": 0.13863492012023926,
+      "logps/chosen": -1.3027557134628296,
+      "logps/rejected": -1.454535722732544,
+      "loss": 1.6514,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.3027557134628296,
+      "rewards/margins": 0.15178008377552032,
+      "rewards/rejected": -1.454535722732544,
+      "step": 510
+    },
+    {
+      "epoch": 0.2756313764843619,
+      "grad_norm": 8.590703247363546,
+      "learning_rate": 9.180035650623885e-07,
+      "logits/chosen": 0.050558000802993774,
+      "logits/rejected": 0.14124636352062225,
+      "logps/chosen": -1.2460167407989502,
+      "logps/rejected": -1.3840495347976685,
+      "loss": 1.6088,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.2460167407989502,
+      "rewards/margins": 0.1380326896905899,
+      "rewards/rejected": -1.3840495347976685,
+      "step": 515
+    },
+    {
+      "epoch": 0.27830740926576353,
+      "grad_norm": 6.893933372683863,
+      "learning_rate": 9.26916221033868e-07,
+      "logits/chosen": -0.08430425822734833,
+      "logits/rejected": 0.04368329793214798,
+      "logps/chosen": -1.2867071628570557,
+      "logps/rejected": -1.3750287294387817,
+      "loss": 1.695,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.2867071628570557,
+      "rewards/margins": 0.08832161128520966,
+      "rewards/rejected": -1.3750287294387817,
+      "step": 520
+    },
+    {
+      "epoch": 0.2809834420471651,
+      "grad_norm": 19.67350727764414,
+      "learning_rate": 9.358288770053476e-07,
+      "logits/chosen": 0.09718038141727448,
+      "logits/rejected": 0.16004881262779236,
+      "logps/chosen": -1.264098882675171,
+      "logps/rejected": -1.4222582578659058,
+      "loss": 1.6412,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.264098882675171,
+      "rewards/margins": 0.1581595242023468,
+      "rewards/rejected": -1.4222582578659058,
+      "step": 525
+    },
+    {
+      "epoch": 0.2836594748285666,
+      "grad_norm": 6.274997373019293,
+      "learning_rate": 9.44741532976827e-07,
+      "logits/chosen": 0.0687488541007042,
+      "logits/rejected": 0.14788378775119781,
+      "logps/chosen": -1.2366673946380615,
+      "logps/rejected": -1.3275758028030396,
+      "loss": 1.6578,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.2366673946380615,
+      "rewards/margins": 0.09090828895568848,
+      "rewards/rejected": -1.3275758028030396,
+      "step": 530
+    },
+    {
+      "epoch": 0.28633550760996823,
+      "grad_norm": 7.0945040486516175,
+      "learning_rate": 9.536541889483066e-07,
+      "logits/chosen": -0.10038735717535019,
+      "logits/rejected": 0.14553984999656677,
+      "logps/chosen": -1.2521215677261353,
+      "logps/rejected": -1.3115968704223633,
+      "loss": 1.6602,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.2521215677261353,
+      "rewards/margins": 0.059475433081388474,
+      "rewards/rejected": -1.3115968704223633,
+      "step": 535
+    },
+    {
+      "epoch": 0.2890115403913698,
+      "grad_norm": 7.789872109139343,
+      "learning_rate": 9.62566844919786e-07,
+      "logits/chosen": 0.016909483820199966,
+      "logits/rejected": 0.08182911574840546,
+      "logps/chosen": -1.378409743309021,
+      "logps/rejected": -1.4266738891601562,
+      "loss": 1.7337,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.378409743309021,
+      "rewards/margins": 0.04826412349939346,
+      "rewards/rejected": -1.4266738891601562,
+      "step": 540
+    },
+    {
+      "epoch": 0.2916875731727714,
+      "grad_norm": 7.359174830858011,
+      "learning_rate": 9.714795008912655e-07,
+      "logits/chosen": -0.10508112609386444,
+      "logits/rejected": 0.08393336832523346,
+      "logps/chosen": -1.2827831506729126,
+      "logps/rejected": -1.3865883350372314,
+      "loss": 1.6549,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.2827831506729126,
+      "rewards/margins": 0.10380531847476959,
+      "rewards/rejected": -1.3865883350372314,
+      "step": 545
+    },
+    {
+      "epoch": 0.29436360595417294,
+      "grad_norm": 8.778259311834221,
+      "learning_rate": 9.80392156862745e-07,
+      "logits/chosen": 0.04168703407049179,
+      "logits/rejected": 0.10389252007007599,
+      "logps/chosen": -1.280343770980835,
+      "logps/rejected": -1.402133822441101,
+      "loss": 1.6367,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.280343770980835,
+      "rewards/margins": 0.12179000675678253,
+      "rewards/rejected": -1.402133822441101,
+      "step": 550
+    },
+    {
+      "epoch": 0.2970396387355745,
+      "grad_norm": 11.267802618540536,
+      "learning_rate": 9.893048128342244e-07,
+      "logits/chosen": -0.07766524702310562,
+      "logits/rejected": 0.04163938760757446,
+      "logps/chosen": -1.356514573097229,
+      "logps/rejected": -1.4130576848983765,
+      "loss": 1.7217,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.356514573097229,
+      "rewards/margins": 0.056543171405792236,
+      "rewards/rejected": -1.4130576848983765,
+      "step": 555
+    },
+    {
+      "epoch": 0.2997156715169761,
+      "grad_norm": 9.734534917386588,
+      "learning_rate": 9.98217468805704e-07,
+      "logits/chosen": 0.0625699833035469,
+      "logits/rejected": 0.07177621126174927,
+      "logps/chosen": -1.2119487524032593,
+      "logps/rejected": -1.33185875415802,
+      "loss": 1.6101,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.2119487524032593,
+      "rewards/margins": 0.11990992724895477,
+      "rewards/rejected": -1.33185875415802,
+      "step": 560
+    },
+    {
+      "epoch": 0.30239170429837764,
+      "grad_norm": 6.1694254592049536,
+      "learning_rate": 9.999984476788462e-07,
+      "logits/chosen": 0.030029237270355225,
+      "logits/rejected": 0.07460276782512665,
+      "logps/chosen": -1.3244996070861816,
+      "logps/rejected": -1.4376353025436401,
+      "loss": 1.6988,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.3244996070861816,
+      "rewards/margins": 0.11313574016094208,
+      "rewards/rejected": -1.4376353025436401,
+      "step": 565
+    },
+    {
+      "epoch": 0.30506773707977924,
+      "grad_norm": 8.854668170263972,
+      "learning_rate": 9.999921413906797e-07,
+      "logits/chosen": -0.04299437254667282,
+      "logits/rejected": 0.1687401980161667,
+      "logps/chosen": -1.3059120178222656,
+      "logps/rejected": -1.37355637550354,
+      "loss": 1.702,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.3059120178222656,
+      "rewards/margins": 0.0676443800330162,
+      "rewards/rejected": -1.37355637550354,
+      "step": 570
+    },
+    {
+      "epoch": 0.3077437698611808,
+      "grad_norm": 6.858926964971097,
+      "learning_rate": 9.999809841765644e-07,
+      "logits/chosen": -0.0256123635917902,
+      "logits/rejected": 0.029277730733156204,
+      "logps/chosen": -1.2328436374664307,
+      "logps/rejected": -1.3388514518737793,
+      "loss": 1.6605,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.2328436374664307,
+      "rewards/margins": 0.1060078889131546,
+      "rewards/rejected": -1.3388514518737793,
+      "step": 575
+    },
+    {
+      "epoch": 0.3104198026425824,
+      "grad_norm": 7.451636113265376,
+      "learning_rate": 9.999649761447477e-07,
+      "logits/chosen": -0.03498077020049095,
+      "logits/rejected": 0.11152464151382446,
+      "logps/chosen": -1.2289667129516602,
+      "logps/rejected": -1.3867911100387573,
+      "loss": 1.5999,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.2289667129516602,
+      "rewards/margins": 0.1578243523836136,
+      "rewards/rejected": -1.3867911100387573,
+      "step": 580
+    },
+    {
+      "epoch": 0.31309583542398395,
+      "grad_norm": 7.454532761341038,
+      "learning_rate": 9.999441174505398e-07,
+      "logits/chosen": -0.07113450020551682,
+      "logits/rejected": 0.022057075053453445,
+      "logps/chosen": -1.3707550764083862,
+      "logps/rejected": -1.412688136100769,
+      "loss": 1.7413,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.3707550764083862,
+      "rewards/margins": 0.04193300008773804,
+      "rewards/rejected": -1.412688136100769,
+      "step": 585
+    },
+    {
+      "epoch": 0.3157718682053855,
+      "grad_norm": 15.759533284704874,
+      "learning_rate": 9.999184082963116e-07,
+      "logits/chosen": -0.02356426976621151,
+      "logits/rejected": 0.09675613045692444,
+      "logps/chosen": -1.3437496423721313,
+      "logps/rejected": -1.3699607849121094,
+      "loss": 1.7185,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.3437496423721313,
+      "rewards/margins": 0.0262112058699131,
+      "rewards/rejected": -1.3699607849121094,
+      "step": 590
+    },
+    {
+      "epoch": 0.3184479009867871,
+      "grad_norm": 8.753436869586725,
+      "learning_rate": 9.998878489314937e-07,
+      "logits/chosen": 0.001271332846954465,
+      "logits/rejected": 0.118097685277462,
+      "logps/chosen": -1.275848150253296,
+      "logps/rejected": -1.3325821161270142,
+      "loss": 1.681,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -1.275848150253296,
+      "rewards/margins": 0.05673399567604065,
+      "rewards/rejected": -1.3325821161270142,
+      "step": 595
+    },
+    {
+      "epoch": 0.32112393376818865,
+      "grad_norm": 8.206758711816612,
+      "learning_rate": 9.99852439652573e-07,
+      "logits/chosen": -0.05460699647665024,
+      "logits/rejected": 0.08076707273721695,
+      "logps/chosen": -1.2695443630218506,
+      "logps/rejected": -1.2824230194091797,
+      "loss": 1.6864,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -1.2695443630218506,
+      "rewards/margins": 0.012878738343715668,
+      "rewards/rejected": -1.2824230194091797,
+      "step": 600
+    },
+    {
+      "epoch": 0.32379996654959026,
+      "grad_norm": 9.827295432839174,
+      "learning_rate": 9.998121808030904e-07,
+      "logits/chosen": -0.08948203921318054,
+      "logits/rejected": -0.007546691689640284,
+      "logps/chosen": -1.330176591873169,
+      "logps/rejected": -1.4655649662017822,
+      "loss": 1.6795,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.330176591873169,
+      "rewards/margins": 0.1353883445262909,
+      "rewards/rejected": -1.4655649662017822,
+      "step": 605
+    },
+    {
+      "epoch": 0.3264759993309918,
+      "grad_norm": 12.364970388660888,
+      "learning_rate": 9.997670727736379e-07,
+      "logits/chosen": 0.0591292604804039,
+      "logits/rejected": 0.2056918442249298,
+      "logps/chosen": -1.308626651763916,
+      "logps/rejected": -1.3645381927490234,
+      "loss": 1.7032,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.308626651763916,
+      "rewards/margins": 0.05591156333684921,
+      "rewards/rejected": -1.3645381927490234,
+      "step": 610
+    },
+    {
+      "epoch": 0.32915203211239336,
+      "grad_norm": 5.644018971156467,
+      "learning_rate": 9.99717116001853e-07,
+      "logits/chosen": -0.05800892040133476,
+      "logits/rejected": 0.04104765132069588,
+      "logps/chosen": -1.3062307834625244,
+      "logps/rejected": -1.432071328163147,
+      "loss": 1.6772,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.3062307834625244,
+      "rewards/margins": 0.1258404701948166,
+      "rewards/rejected": -1.432071328163147,
+      "step": 615
+    },
+    {
+      "epoch": 0.33182806489379496,
+      "grad_norm": 7.3898691520823725,
+      "learning_rate": 9.996623109724173e-07,
+      "logits/chosen": 0.05058063194155693,
+      "logits/rejected": 0.11339068412780762,
+      "logps/chosen": -1.3660705089569092,
+      "logps/rejected": -1.4618982076644897,
+      "loss": 1.7012,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -1.3660705089569092,
+      "rewards/margins": 0.09582777321338654,
+      "rewards/rejected": -1.4618982076644897,
+      "step": 620
+    },
+    {
+      "epoch": 0.3345040976751965,
+      "grad_norm": 8.959693793832603,
+      "learning_rate": 9.996026582170488e-07,
+      "logits/chosen": 0.06929932534694672,
+      "logits/rejected": 0.1785743236541748,
+      "logps/chosen": -1.2847315073013306,
+      "logps/rejected": -1.4012258052825928,
+      "loss": 1.6582,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.2847315073013306,
+      "rewards/margins": 0.11649449169635773,
+      "rewards/rejected": -1.4012258052825928,
+      "step": 625
+    },
+    {
+      "epoch": 0.3371801304565981,
+      "grad_norm": 7.628006754940694,
+      "learning_rate": 9.995381583144996e-07,
+      "logits/chosen": -0.01316717267036438,
+      "logits/rejected": 0.08945786952972412,
+      "logps/chosen": -1.3198153972625732,
+      "logps/rejected": -1.448467493057251,
+      "loss": 1.6546,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.3198153972625732,
+      "rewards/margins": 0.12865200638771057,
+      "rewards/rejected": -1.448467493057251,
+      "step": 630
+    },
+    {
+      "epoch": 0.33985616323799966,
+      "grad_norm": 6.656545125521265,
+      "learning_rate": 9.994688118905471e-07,
+      "logits/chosen": -0.012319590896368027,
+      "logits/rejected": 0.21458666026592255,
+      "logps/chosen": -1.3832319974899292,
+      "logps/rejected": -1.4100427627563477,
+      "loss": 1.7693,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -1.3832319974899292,
+      "rewards/margins": 0.0268106609582901,
+      "rewards/rejected": -1.4100427627563477,
+      "step": 635
+    },
+    {
+      "epoch": 0.3425321960194012,
+      "grad_norm": 8.970597384974177,
+      "learning_rate": 9.993946196179912e-07,
+      "logits/chosen": -0.08106163144111633,
+      "logits/rejected": 0.10695508867502213,
+      "logps/chosen": -1.3231474161148071,
+      "logps/rejected": -1.4096291065216064,
+      "loss": 1.7402,
+      "rewards/accuracies": 0.45625001192092896,
+      "rewards/chosen": -1.3231474161148071,
+      "rewards/margins": 0.08648180961608887,
+      "rewards/rejected": -1.4096291065216064,
+      "step": 640
+    },
+    {
+      "epoch": 0.3452082288008028,
+      "grad_norm": 7.346230858532709,
+      "learning_rate": 9.993155822166455e-07,
+      "logits/chosen": -0.08081404119729996,
+      "logits/rejected": 0.0013829886447638273,
+      "logps/chosen": -1.2329657077789307,
+      "logps/rejected": -1.3853070735931396,
+      "loss": 1.6505,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.2329657077789307,
+      "rewards/margins": 0.1523413360118866,
+      "rewards/rejected": -1.3853070735931396,
+      "step": 645
+    },
+    {
+      "epoch": 0.34788426158220437,
+      "grad_norm": 10.907122686264634,
+      "learning_rate": 9.992317004533313e-07,
+      "logits/chosen": -0.04056433588266373,
+      "logits/rejected": 0.09426826983690262,
+      "logps/chosen": -1.3744598627090454,
+      "logps/rejected": -1.517823338508606,
+      "loss": 1.713,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.3744598627090454,
+      "rewards/margins": 0.14336350560188293,
+      "rewards/rejected": -1.517823338508606,
+      "step": 650
+    },
+    {
+      "epoch": 0.350560294363606,
+      "grad_norm": 8.246432749787882,
+      "learning_rate": 9.991429751418696e-07,
+      "logits/chosen": 0.024029621854424477,
+      "logits/rejected": 0.028746861964464188,
+      "logps/chosen": -1.3122899532318115,
+      "logps/rejected": -1.4840376377105713,
+      "loss": 1.6704,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.3122899532318115,
+      "rewards/margins": 0.17174772918224335,
+      "rewards/rejected": -1.4840376377105713,
+      "step": 655
+    },
+    {
+      "epoch": 0.3532363271450075,
+      "grad_norm": 7.0970140924944225,
+      "learning_rate": 9.99049407143074e-07,
+      "logits/chosen": 0.014008665457367897,
+      "logits/rejected": 0.13609251379966736,
+      "logps/chosen": -1.270634412765503,
+      "logps/rejected": -1.2979482412338257,
+      "loss": 1.6808,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.270634412765503,
+      "rewards/margins": 0.027314016595482826,
+      "rewards/rejected": -1.2979482412338257,
+      "step": 660
+    },
+    {
+      "epoch": 0.35591235992640907,
+      "grad_norm": 7.628095065228471,
+      "learning_rate": 9.989509973647416e-07,
+      "logits/chosen": -0.008538028225302696,
+      "logits/rejected": 0.12382189929485321,
+      "logps/chosen": -1.2361500263214111,
+      "logps/rejected": -1.3574538230895996,
+      "loss": 1.6301,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.2361500263214111,
+      "rewards/margins": 0.1213037520647049,
+      "rewards/rejected": -1.3574538230895996,
+      "step": 665
+    },
+    {
+      "epoch": 0.3585883927078107,
+      "grad_norm": 8.552725286884353,
+      "learning_rate": 9.988477467616445e-07,
+      "logits/chosen": -0.02346823923289776,
+      "logits/rejected": 0.17321622371673584,
+      "logps/chosen": -1.2735487222671509,
+      "logps/rejected": -1.3107669353485107,
+      "loss": 1.669,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -1.2735487222671509,
+      "rewards/margins": 0.0372183695435524,
+      "rewards/rejected": -1.3107669353485107,
+      "step": 670
+    },
+    {
+      "epoch": 0.3612644254892122,
+      "grad_norm": 10.528437312230285,
+      "learning_rate": 9.987396563355205e-07,
+      "logits/chosen": -0.036085594445466995,
+      "logits/rejected": 0.03549571335315704,
+      "logps/chosen": -1.2650729417800903,
+      "logps/rejected": -1.4736168384552002,
+      "loss": 1.6303,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.2650729417800903,
+      "rewards/margins": 0.20854397118091583,
+      "rewards/rejected": -1.4736168384552002,
+      "step": 675
+    },
+    {
+      "epoch": 0.36394045827061383,
+      "grad_norm": 9.271902916867035,
+      "learning_rate": 9.986267271350631e-07,
+      "logits/chosen": 0.06981401890516281,
+      "logits/rejected": 0.22414882481098175,
+      "logps/chosen": -1.3095825910568237,
+      "logps/rejected": -1.3565170764923096,
+      "loss": 1.7206,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -1.3095825910568237,
+      "rewards/margins": 0.04693450406193733,
+      "rewards/rejected": -1.3565170764923096,
+      "step": 680
+    },
+    {
+      "epoch": 0.3666164910520154,
+      "grad_norm": 9.197512258817639,
+      "learning_rate": 9.985089602559123e-07,
+      "logits/chosen": 0.020719774067401886,
+      "logits/rejected": 0.17003081738948822,
+      "logps/chosen": -1.2913005352020264,
+      "logps/rejected": -1.3424879312515259,
+      "loss": 1.6958,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.2913005352020264,
+      "rewards/margins": 0.051187384873628616,
+      "rewards/rejected": -1.3424879312515259,
+      "step": 685
+    },
+    {
+      "epoch": 0.369292523833417,
+      "grad_norm": 8.633540226735317,
+      "learning_rate": 9.983863568406428e-07,
+      "logits/chosen": 0.03548329323530197,
+      "logits/rejected": 0.0648711770772934,
+      "logps/chosen": -1.2927361726760864,
+      "logps/rejected": -1.4053599834442139,
+      "loss": 1.6573,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.2927361726760864,
+      "rewards/margins": 0.11262372881174088,
+      "rewards/rejected": -1.4053599834442139,
+      "step": 690
+    },
+    {
+      "epoch": 0.37196855661481854,
+      "grad_norm": 7.219278802274095,
+      "learning_rate": 9.982589180787532e-07,
+      "logits/chosen": -0.00833642203360796,
+      "logits/rejected": 0.07940270006656647,
+      "logps/chosen": -1.1803183555603027,
+      "logps/rejected": -1.3500235080718994,
+      "loss": 1.5644,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.1803183555603027,
+      "rewards/margins": 0.16970540583133698,
+      "rewards/rejected": -1.3500235080718994,
+      "step": 695
+    },
+    {
+      "epoch": 0.3746445893962201,
+      "grad_norm": 6.240252885623886,
+      "learning_rate": 9.981266452066553e-07,
+      "logits/chosen": -0.10447759926319122,
+      "logits/rejected": 0.024963444098830223,
+      "logps/chosen": -1.3531849384307861,
+      "logps/rejected": -1.4271669387817383,
+      "loss": 1.7094,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.3531849384307861,
+      "rewards/margins": 0.07398195564746857,
+      "rewards/rejected": -1.4271669387817383,
+      "step": 700
+    },
+    {
+      "epoch": 0.3773206221776217,
+      "grad_norm": 6.504425773436158,
+      "learning_rate": 9.979895395076608e-07,
+      "logits/chosen": -0.09504284709692001,
+      "logits/rejected": 0.06818296760320663,
+      "logps/chosen": -1.3101884126663208,
+      "logps/rejected": -1.4512856006622314,
+      "loss": 1.6811,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.3101884126663208,
+      "rewards/margins": 0.1410975158214569,
+      "rewards/rejected": -1.4512856006622314,
+      "step": 705
+    },
+    {
+      "epoch": 0.37999665495902324,
+      "grad_norm": 7.997459613296311,
+      "learning_rate": 9.9784760231197e-07,
+      "logits/chosen": 0.028507554903626442,
+      "logits/rejected": 0.1171996220946312,
+      "logps/chosen": -1.2455408573150635,
+      "logps/rejected": -1.3672510385513306,
+      "loss": 1.6293,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.2455408573150635,
+      "rewards/margins": 0.12171021848917007,
+      "rewards/rejected": -1.3672510385513306,
+      "step": 710
+    },
+    {
+      "epoch": 0.38267268774042484,
+      "grad_norm": 8.537076673133495,
+      "learning_rate": 9.97700834996658e-07,
+      "logits/chosen": -0.02836451306939125,
+      "logits/rejected": 0.126664400100708,
+      "logps/chosen": -1.3226584196090698,
+      "logps/rejected": -1.4415714740753174,
+      "loss": 1.7028,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.3226584196090698,
+      "rewards/margins": 0.11891325563192368,
+      "rewards/rejected": -1.4415714740753174,
+      "step": 715
+    },
+    {
+      "epoch": 0.3853487205218264,
+      "grad_norm": 7.544234541581894,
+      "learning_rate": 9.97549238985662e-07,
+      "logits/chosen": 0.041222043335437775,
+      "logits/rejected": 0.2183455228805542,
+      "logps/chosen": -1.3687158823013306,
+      "logps/rejected": -1.4198092222213745,
+      "loss": 1.7279,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.3687158823013306,
+      "rewards/margins": 0.05109328031539917,
+      "rewards/rejected": -1.4198092222213745,
+      "step": 720
+    },
+    {
+      "epoch": 0.38802475330322794,
+      "grad_norm": 9.122685189648141,
+      "learning_rate": 9.973928157497674e-07,
+      "logits/chosen": -0.0802018791437149,
+      "logits/rejected": 0.049556490033864975,
+      "logps/chosen": -1.2097431421279907,
+      "logps/rejected": -1.4569720029830933,
+      "loss": 1.5884,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.2097431421279907,
+      "rewards/margins": 0.24722866714000702,
+      "rewards/rejected": -1.4569720029830933,
+      "step": 725
+    },
+    {
+      "epoch": 0.39070078608462955,
+      "grad_norm": 9.481217491829481,
+      "learning_rate": 9.972315668065927e-07,
+      "logits/chosen": -0.09887029230594635,
+      "logits/rejected": 0.06117262318730354,
+      "logps/chosen": -1.3061532974243164,
+      "logps/rejected": -1.3930120468139648,
+      "loss": 1.6769,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.3061532974243164,
+      "rewards/margins": 0.08685862272977829,
+      "rewards/rejected": -1.3930120468139648,
+      "step": 730
+    },
+    {
+      "epoch": 0.3933768188660311,
+      "grad_norm": 6.656490793599791,
+      "learning_rate": 9.97065493720576e-07,
+      "logits/chosen": -0.0811881572008133,
+      "logits/rejected": 0.014002988114953041,
+      "logps/chosen": -1.328331708908081,
+      "logps/rejected": -1.384516716003418,
+      "loss": 1.6862,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.328331708908081,
+      "rewards/margins": 0.05618508532643318,
+      "rewards/rejected": -1.384516716003418,
+      "step": 735
+    },
+    {
+      "epoch": 0.3960528516474327,
+      "grad_norm": 10.559224036803714,
+      "learning_rate": 9.968945981029594e-07,
+      "logits/chosen": -0.04994923993945122,
+      "logits/rejected": 0.1147971898317337,
+      "logps/chosen": -1.3782548904418945,
+      "logps/rejected": -1.4143259525299072,
+      "loss": 1.748,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -1.3782548904418945,
+      "rewards/margins": 0.03607102110981941,
+      "rewards/rejected": -1.4143259525299072,
+      "step": 740
+    },
+    {
+      "epoch": 0.39872888442883425,
+      "grad_norm": 7.189987022376075,
+      "learning_rate": 9.967188816117726e-07,
+      "logits/chosen": 0.036188650876283646,
+      "logits/rejected": 0.09928051382303238,
+      "logps/chosen": -1.359431266784668,
+      "logps/rejected": -1.5481358766555786,
+      "loss": 1.7021,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.359431266784668,
+      "rewards/margins": 0.18870458006858826,
+      "rewards/rejected": -1.5481358766555786,
+      "step": 745
+    },
+    {
+      "epoch": 0.4014049172102358,
+      "grad_norm": 6.989374751704205,
+      "learning_rate": 9.965383459518179e-07,
+      "logits/chosen": -0.022238023579120636,
+      "logits/rejected": 0.13665586709976196,
+      "logps/chosen": -1.2956759929656982,
+      "logps/rejected": -1.4461919069290161,
+      "loss": 1.6496,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.2956759929656982,
+      "rewards/margins": 0.15051576495170593,
+      "rewards/rejected": -1.4461919069290161,
+      "step": 750
+    },
+    {
+      "epoch": 0.4040809499916374,
+      "grad_norm": 5.70105952630306,
+      "learning_rate": 9.963529928746533e-07,
+      "logits/chosen": 0.004465815611183643,
+      "logits/rejected": 0.12207271158695221,
+      "logps/chosen": -1.327978491783142,
+      "logps/rejected": -1.3998768329620361,
+      "loss": 1.6933,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.327978491783142,
+      "rewards/margins": 0.07189829647541046,
+      "rewards/rejected": -1.3998768329620361,
+      "step": 755
+    },
+    {
+      "epoch": 0.40675698277303896,
+      "grad_norm": 5.94205764786591,
+      "learning_rate": 9.961628241785746e-07,
+      "logits/chosen": -0.06844659149646759,
+      "logits/rejected": -0.002805382013320923,
+      "logps/chosen": -1.3424692153930664,
+      "logps/rejected": -1.4756808280944824,
+      "loss": 1.6882,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.3424692153930664,
+      "rewards/margins": 0.13321153819561005,
+      "rewards/rejected": -1.4756808280944824,
+      "step": 760
+    },
+    {
+      "epoch": 0.40943301555444056,
+      "grad_norm": 14.34219789446751,
+      "learning_rate": 9.959678417085998e-07,
+      "logits/chosen": -0.04711277782917023,
+      "logits/rejected": 0.03717537969350815,
+      "logps/chosen": -1.2987074851989746,
+      "logps/rejected": -1.3974730968475342,
+      "loss": 1.659,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.2987074851989746,
+      "rewards/margins": 0.09876558184623718,
+      "rewards/rejected": -1.3974730968475342,
+      "step": 765
+    },
+    {
+      "epoch": 0.4121090483358421,
+      "grad_norm": 7.668216776232713,
+      "learning_rate": 9.957680473564493e-07,
+      "logits/chosen": 0.07918961346149445,
+      "logits/rejected": 0.1951749622821808,
+      "logps/chosen": -1.257340669631958,
+      "logps/rejected": -1.4603685140609741,
+      "loss": 1.6128,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.257340669631958,
+      "rewards/margins": 0.2030281275510788,
+      "rewards/rejected": -1.4603685140609741,
+      "step": 770
+    },
+    {
+      "epoch": 0.41478508111724366,
+      "grad_norm": 9.20239361618736,
+      "learning_rate": 9.95563443060529e-07,
+      "logits/chosen": -0.07869169116020203,
+      "logits/rejected": 0.08225543797016144,
+      "logps/chosen": -1.3372876644134521,
+      "logps/rejected": -1.5053128004074097,
+      "loss": 1.6958,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.3372876644134521,
+      "rewards/margins": 0.16802528500556946,
+      "rewards/rejected": -1.5053128004074097,
+      "step": 775
+    },
+    {
+      "epoch": 0.41746111389864526,
+      "grad_norm": 7.2244031080730755,
+      "learning_rate": 9.95354030805911e-07,
+      "logits/chosen": -0.1303524672985077,
+      "logits/rejected": 0.009751476347446442,
+      "logps/chosen": -1.2609707117080688,
+      "logps/rejected": -1.414001226425171,
+      "loss": 1.6255,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.2609707117080688,
+      "rewards/margins": 0.1530303955078125,
+      "rewards/rejected": -1.414001226425171,
+      "step": 780
+    },
+    {
+      "epoch": 0.4201371466800468,
+      "grad_norm": 8.457832296987833,
+      "learning_rate": 9.951398126243133e-07,
+      "logits/chosen": 0.022420603781938553,
+      "logits/rejected": 0.1414371132850647,
+      "logps/chosen": -1.2476838827133179,
+      "logps/rejected": -1.4334955215454102,
+      "loss": 1.6246,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.2476838827133179,
+      "rewards/margins": 0.18581156432628632,
+      "rewards/rejected": -1.4334955215454102,
+      "step": 785
+    },
+    {
+      "epoch": 0.4228131794614484,
+      "grad_norm": 7.083834282044994,
+      "learning_rate": 9.94920790594082e-07,
+      "logits/chosen": -0.04662860929965973,
+      "logits/rejected": 0.06980902701616287,
+      "logps/chosen": -1.3007665872573853,
+      "logps/rejected": -1.3650496006011963,
+      "loss": 1.6828,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.3007665872573853,
+      "rewards/margins": 0.0642831102013588,
+      "rewards/rejected": -1.3650496006011963,
+      "step": 790
+    },
+    {
+      "epoch": 0.42548921224284997,
+      "grad_norm": 5.925629002132357,
+      "learning_rate": 9.946969668401696e-07,
+      "logits/chosen": -0.07388118654489517,
+      "logits/rejected": 0.1055014580488205,
+      "logps/chosen": -1.2804076671600342,
+      "logps/rejected": -1.4107162952423096,
+      "loss": 1.6566,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.2804076671600342,
+      "rewards/margins": 0.1303088366985321,
+      "rewards/rejected": -1.4107162952423096,
+      "step": 795
+    },
+    {
+      "epoch": 0.4281652450242516,
+      "grad_norm": 7.440409871322956,
+      "learning_rate": 9.944683435341155e-07,
+      "logits/chosen": -0.029027258977293968,
+      "logits/rejected": 0.046039216220378876,
+      "logps/chosen": -1.2842944860458374,
+      "logps/rejected": -1.3327921628952026,
+      "loss": 1.6693,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.2842944860458374,
+      "rewards/margins": 0.048497579991817474,
+      "rewards/rejected": -1.3327921628952026,
+      "step": 800
+    },
+    {
+      "epoch": 0.4281652450242516,
+      "eval_logits/chosen": 0.2642241418361664,
+      "eval_logits/rejected": 0.3440663516521454,
+      "eval_logps/chosen": -1.3149185180664062,
+      "eval_logps/rejected": -1.4532232284545898,
+      "eval_loss": 1.671885371208191,
+      "eval_rewards/accuracies": 0.5578634738922119,
+      "eval_rewards/chosen": -1.3149185180664062,
+      "eval_rewards/margins": 0.13830474019050598,
+      "eval_rewards/rejected": -1.4532232284545898,
+      "eval_runtime": 40.5458,
+      "eval_samples_per_second": 33.172,
+      "eval_steps_per_second": 8.312,
+      "step": 800
+    },
+    {
+      "epoch": 0.4308412778056531,
+      "grad_norm": 7.508015672817706,
+      "learning_rate": 9.942349228940236e-07,
+      "logits/chosen": -0.0667061060667038,
+      "logits/rejected": 0.07563512772321701,
+      "logps/chosen": -1.3263849020004272,
+      "logps/rejected": -1.476535439491272,
+      "loss": 1.68,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.3263849020004272,
+      "rewards/margins": 0.1501503884792328,
+      "rewards/rejected": -1.476535439491272,
+      "step": 805
+    },
+    {
+      "epoch": 0.43351731058705467,
+      "grad_norm": 7.655604039956669,
+      "learning_rate": 9.939967071845424e-07,
+      "logits/chosen": 0.02369997464120388,
+      "logits/rejected": 0.09098490327596664,
+      "logps/chosen": -1.2372820377349854,
+      "logps/rejected": -1.3665462732315063,
+      "loss": 1.6153,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.2372820377349854,
+      "rewards/margins": 0.12926435470581055,
+      "rewards/rejected": -1.3665462732315063,
+      "step": 810
+    },
+    {
+      "epoch": 0.4361933433684563,
+      "grad_norm": 8.11934052284553,
+      "learning_rate": 9.937536987168413e-07,
+      "logits/chosen": 0.04012486711144447,
+      "logits/rejected": 0.1624753475189209,
+      "logps/chosen": -1.2311725616455078,
+      "logps/rejected": -1.4302477836608887,
+      "loss": 1.5796,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.2311725616455078,
+      "rewards/margins": 0.19907523691654205,
+      "rewards/rejected": -1.4302477836608887,
+      "step": 815
+    },
+    {
+      "epoch": 0.4388693761498578,
+      "grad_norm": 7.76861185601798,
+      "learning_rate": 9.935058998485896e-07,
+      "logits/chosen": 0.035312581807374954,
+      "logits/rejected": 0.0773046463727951,
+      "logps/chosen": -1.277048110961914,
+      "logps/rejected": -1.4330211877822876,
+      "loss": 1.6463,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.277048110961914,
+      "rewards/margins": 0.15597307682037354,
+      "rewards/rejected": -1.4330211877822876,
+      "step": 820
+    },
+    {
+      "epoch": 0.44154540893125943,
+      "grad_norm": 7.860471019420282,
+      "learning_rate": 9.932533129839333e-07,
+      "logits/chosen": -0.014185379259288311,
+      "logits/rejected": 0.10354729741811752,
+      "logps/chosen": -1.2197961807250977,
+      "logps/rejected": -1.306098222732544,
+      "loss": 1.6297,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -1.2197961807250977,
+      "rewards/margins": 0.08630184829235077,
+      "rewards/rejected": -1.306098222732544,
+      "step": 825
+    },
+    {
+      "epoch": 0.444221441712661,
+      "grad_norm": 6.980346415002472,
+      "learning_rate": 9.929959405734711e-07,
+      "logits/chosen": 0.044074997305870056,
+      "logits/rejected": 0.19551844894886017,
+      "logps/chosen": -1.3337340354919434,
+      "logps/rejected": -1.3899133205413818,
+      "loss": 1.7003,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.3337340354919434,
+      "rewards/margins": 0.0561792366206646,
+      "rewards/rejected": -1.3899133205413818,
+      "step": 830
+    },
+    {
+      "epoch": 0.44689747449406253,
+      "grad_norm": 10.643519397721393,
+      "learning_rate": 9.927337851142314e-07,
+      "logits/chosen": -0.013499056920409203,
+      "logits/rejected": 0.10668818652629852,
+      "logps/chosen": -1.2430568933486938,
+      "logps/rejected": -1.3656256198883057,
+      "loss": 1.6498,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.2430568933486938,
+      "rewards/margins": 0.12256884574890137,
+      "rewards/rejected": -1.3656256198883057,
+      "step": 835
+    },
+    {
+      "epoch": 0.44957350727546413,
+      "grad_norm": 8.177028406683556,
+      "learning_rate": 9.924668491496474e-07,
+      "logits/chosen": -0.004524660296738148,
+      "logits/rejected": 0.1376916915178299,
+      "logps/chosen": -1.284436583518982,
+      "logps/rejected": -1.4487638473510742,
+      "loss": 1.666,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.284436583518982,
+      "rewards/margins": 0.1643272340297699,
+      "rewards/rejected": -1.4487638473510742,
+      "step": 840
+    },
+    {
+      "epoch": 0.4522495400568657,
+      "grad_norm": 5.537721258154138,
+      "learning_rate": 9.92195135269533e-07,
+      "logits/chosen": 0.06299177557229996,
+      "logits/rejected": 0.12591898441314697,
+      "logps/chosen": -1.288023829460144,
+      "logps/rejected": -1.341961145401001,
+      "loss": 1.6896,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.288023829460144,
+      "rewards/margins": 0.05393745377659798,
+      "rewards/rejected": -1.341961145401001,
+      "step": 845
+    },
+    {
+      "epoch": 0.4549255728382673,
+      "grad_norm": 9.106385983775208,
+      "learning_rate": 9.919186461100574e-07,
+      "logits/chosen": 0.0094298105686903,
+      "logits/rejected": 0.06498949229717255,
+      "logps/chosen": -1.248669147491455,
+      "logps/rejected": -1.3809727430343628,
+      "loss": 1.6115,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.248669147491455,
+      "rewards/margins": 0.13230359554290771,
+      "rewards/rejected": -1.3809727430343628,
+      "step": 850
+    },
+    {
+      "epoch": 0.45760160561966884,
+      "grad_norm": 6.43224040221898,
+      "learning_rate": 9.9163738435372e-07,
+      "logits/chosen": -0.04182653874158859,
+      "logits/rejected": 0.084588423371315,
+      "logps/chosen": -1.3051223754882812,
+      "logps/rejected": -1.4761320352554321,
+      "loss": 1.6954,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.3051223754882812,
+      "rewards/margins": 0.17100976407527924,
+      "rewards/rejected": -1.4761320352554321,
+      "step": 855
+    },
+    {
+      "epoch": 0.4602776384010704,
+      "grad_norm": 5.868384556976196,
+      "learning_rate": 9.913513527293234e-07,
+      "logits/chosen": -0.06548645347356796,
+      "logits/rejected": 0.07703325897455215,
+      "logps/chosen": -1.3435872793197632,
+      "logps/rejected": -1.5250961780548096,
+      "loss": 1.6786,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.3435872793197632,
+      "rewards/margins": 0.1815088838338852,
+      "rewards/rejected": -1.5250961780548096,
+      "step": 860
+    },
+    {
+      "epoch": 0.462953671182472,
+      "grad_norm": 10.824301867524007,
+      "learning_rate": 9.910605540119474e-07,
+      "logits/chosen": -0.002961373422294855,
+      "logits/rejected": 0.08694703131914139,
+      "logps/chosen": -1.2438809871673584,
+      "logps/rejected": -1.4252790212631226,
+      "loss": 1.6348,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.2438809871673584,
+      "rewards/margins": 0.18139800429344177,
+      "rewards/rejected": -1.4252790212631226,
+      "step": 865
+    },
+    {
+      "epoch": 0.46562970396387354,
+      "grad_norm": 7.012684753512225,
+      "learning_rate": 9.907649910229227e-07,
+      "logits/chosen": -0.08534906059503555,
+      "logits/rejected": 0.15127788484096527,
+      "logps/chosen": -1.3012317419052124,
+      "logps/rejected": -1.3915889263153076,
+      "loss": 1.676,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.3012317419052124,
+      "rewards/margins": 0.09035725891590118,
+      "rewards/rejected": -1.3915889263153076,
+      "step": 870
+    },
+    {
+      "epoch": 0.46830573674527515,
+      "grad_norm": 7.511187500576787,
+      "learning_rate": 9.90464666629803e-07,
+      "logits/chosen": 0.022366385906934738,
+      "logits/rejected": 0.08713110536336899,
+      "logps/chosen": -1.3248341083526611,
+      "logps/rejected": -1.4425128698349,
+      "loss": 1.7236,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.3248341083526611,
+      "rewards/margins": 0.11767883598804474,
+      "rewards/rejected": -1.4425128698349,
+      "step": 875
+    },
+    {
+      "epoch": 0.4709817695266767,
+      "grad_norm": 6.033858000811316,
+      "learning_rate": 9.901595837463363e-07,
+      "logits/chosen": 0.012679760344326496,
+      "logits/rejected": 0.15422797203063965,
+      "logps/chosen": -1.3769097328186035,
+      "logps/rejected": -1.4846652746200562,
+      "loss": 1.7246,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.3769097328186035,
+      "rewards/margins": 0.10775542259216309,
+      "rewards/rejected": -1.4846652746200562,
+      "step": 880
+    },
+    {
+      "epoch": 0.47365780230807825,
+      "grad_norm": 8.760506894298192,
+      "learning_rate": 9.898497453324384e-07,
+      "logits/chosen": -0.054704517126083374,
+      "logits/rejected": 0.016666647046804428,
+      "logps/chosen": -1.2689878940582275,
+      "logps/rejected": -1.4531781673431396,
+      "loss": 1.6311,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.2689878940582275,
+      "rewards/margins": 0.18419024348258972,
+      "rewards/rejected": -1.4531781673431396,
+      "step": 885
+    },
+    {
+      "epoch": 0.47633383508947985,
+      "grad_norm": 7.6920383686095795,
+      "learning_rate": 9.895351543941628e-07,
+      "logits/chosen": -0.14391431212425232,
+      "logits/rejected": -0.03333331272006035,
+      "logps/chosen": -1.318817377090454,
+      "logps/rejected": -1.4459335803985596,
+      "loss": 1.6889,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.318817377090454,
+      "rewards/margins": 0.1271161586046219,
+      "rewards/rejected": -1.4459335803985596,
+      "step": 890
+    },
+    {
+      "epoch": 0.4790098678708814,
+      "grad_norm": 7.794206463550445,
+      "learning_rate": 9.892158139836724e-07,
+      "logits/chosen": 0.05477297306060791,
+      "logits/rejected": 0.16170786321163177,
+      "logps/chosen": -1.2167054414749146,
+      "logps/rejected": -1.3180320262908936,
+      "loss": 1.6156,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.2167054414749146,
+      "rewards/margins": 0.10132656991481781,
+      "rewards/rejected": -1.3180320262908936,
+      "step": 895
+    },
+    {
+      "epoch": 0.481685900652283,
+      "grad_norm": 7.239663409256137,
+      "learning_rate": 9.88891727199209e-07,
+      "logits/chosen": -0.06828771531581879,
+      "logits/rejected": -0.012270675972104073,
+      "logps/chosen": -1.2079464197158813,
+      "logps/rejected": -1.4431251287460327,
+      "loss": 1.5744,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.2079464197158813,
+      "rewards/margins": 0.23517878353595734,
+      "rewards/rejected": -1.4431251287460327,
+      "step": 900
+    },
+    {
+      "epoch": 0.48436193343368455,
+      "grad_norm": 7.101761732783572,
+      "learning_rate": 9.885628971850641e-07,
+      "logits/chosen": 0.034347210079431534,
+      "logits/rejected": 0.20339736342430115,
+      "logps/chosen": -1.2839748859405518,
+      "logps/rejected": -1.4656829833984375,
+      "loss": 1.6507,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.2839748859405518,
+      "rewards/margins": 0.18170809745788574,
+      "rewards/rejected": -1.4656829833984375,
+      "step": 905
+    },
+    {
+      "epoch": 0.48703796621508616,
+      "grad_norm": 5.129875399961622,
+      "learning_rate": 9.882293271315481e-07,
+      "logits/chosen": -0.022476624697446823,
+      "logits/rejected": 0.08038485795259476,
+      "logps/chosen": -1.3369861841201782,
+      "logps/rejected": -1.4090746641159058,
+      "loss": 1.7334,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.3369861841201782,
+      "rewards/margins": 0.07208836078643799,
+      "rewards/rejected": -1.4090746641159058,
+      "step": 910
+    },
+    {
+      "epoch": 0.4897139989964877,
+      "grad_norm": 6.423648732496239,
+      "learning_rate": 9.878910202749589e-07,
+      "logits/chosen": 0.00816698931157589,
+      "logits/rejected": 0.1703716367483139,
+      "logps/chosen": -1.2553752660751343,
+      "logps/rejected": -1.3933888673782349,
+      "loss": 1.6275,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.2553752660751343,
+      "rewards/margins": 0.1380135416984558,
+      "rewards/rejected": -1.3933888673782349,
+      "step": 915
+    },
+    {
+      "epoch": 0.49239003177788926,
+      "grad_norm": 9.911753176821456,
+      "learning_rate": 9.875479798975512e-07,
+      "logits/chosen": 0.1201072558760643,
+      "logits/rejected": 0.2435188591480255,
+      "logps/chosen": -1.221426248550415,
+      "logps/rejected": -1.3659164905548096,
+      "loss": 1.6165,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.221426248550415,
+      "rewards/margins": 0.1444903314113617,
+      "rewards/rejected": -1.3659164905548096,
+      "step": 920
+    },
+    {
+      "epoch": 0.49506606455929086,
+      "grad_norm": 6.945541841571725,
+      "learning_rate": 9.87200209327504e-07,
+      "logits/chosen": -0.014080618508160114,
+      "logits/rejected": 0.1393960863351822,
+      "logps/chosen": -1.305490255355835,
+      "logps/rejected": -1.3544299602508545,
+      "loss": 1.6717,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.305490255355835,
+      "rewards/margins": 0.048939622938632965,
+      "rewards/rejected": -1.3544299602508545,
+      "step": 925
+    },
+    {
+      "epoch": 0.4977420973406924,
+      "grad_norm": 11.57724749033797,
+      "learning_rate": 9.868477119388894e-07,
+      "logits/chosen": -0.054570745676755905,
+      "logits/rejected": 0.04997735470533371,
+      "logps/chosen": -1.2871347665786743,
+      "logps/rejected": -1.4756519794464111,
+      "loss": 1.6784,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.2871347665786743,
+      "rewards/margins": 0.18851712346076965,
+      "rewards/rejected": -1.4756519794464111,
+      "step": 930
+    },
+    {
+      "epoch": 0.500418130122094,
+      "grad_norm": 8.932820080999921,
+      "learning_rate": 9.864904911516383e-07,
+      "logits/chosen": 0.04412764683365822,
+      "logits/rejected": 0.07467509061098099,
+      "logps/chosen": -1.2148029804229736,
+      "logps/rejected": -1.4018632173538208,
+      "loss": 1.6217,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.2148029804229736,
+      "rewards/margins": 0.18706026673316956,
+      "rewards/rejected": -1.4018632173538208,
+      "step": 935
+    },
+    {
+      "epoch": 0.5030941629034956,
+      "grad_norm": 8.357914495043543,
+      "learning_rate": 9.861285504315084e-07,
+      "logits/chosen": -0.028966009616851807,
+      "logits/rejected": 0.07452841848134995,
+      "logps/chosen": -1.2796359062194824,
+      "logps/rejected": -1.3434219360351562,
+      "loss": 1.6655,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.2796359062194824,
+      "rewards/margins": 0.06378618627786636,
+      "rewards/rejected": -1.3434219360351562,
+      "step": 940
+    },
+    {
+      "epoch": 0.5057701956848971,
+      "grad_norm": 7.554214115567587,
+      "learning_rate": 9.857618932900502e-07,
+      "logits/chosen": -0.0378776416182518,
+      "logits/rejected": 0.07617872953414917,
+      "logps/chosen": -1.2562286853790283,
+      "logps/rejected": -1.412217617034912,
+      "loss": 1.6175,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.2562286853790283,
+      "rewards/margins": 0.15598897635936737,
+      "rewards/rejected": -1.412217617034912,
+      "step": 945
+    },
+    {
+      "epoch": 0.5084462284662987,
+      "grad_norm": 6.11351422417312,
+      "learning_rate": 9.853905232845727e-07,
+      "logits/chosen": -0.016596803441643715,
+      "logits/rejected": 0.13409195840358734,
+      "logps/chosen": -1.3554424047470093,
+      "logps/rejected": -1.384244680404663,
+      "loss": 1.7487,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -1.3554424047470093,
+      "rewards/margins": 0.028802240267395973,
+      "rewards/rejected": -1.384244680404663,
+      "step": 950
+    },
+    {
+      "epoch": 0.5111222612477003,
+      "grad_norm": 5.893037811337725,
+      "learning_rate": 9.850144440181095e-07,
+      "logits/chosen": -0.007894618436694145,
+      "logits/rejected": 0.18766812980175018,
+      "logps/chosen": -1.3416087627410889,
+      "logps/rejected": -1.3942476511001587,
+      "loss": 1.738,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -1.3416087627410889,
+      "rewards/margins": 0.052638787776231766,
+      "rewards/rejected": -1.3942476511001587,
+      "step": 955
+    },
+    {
+      "epoch": 0.5137982940291018,
+      "grad_norm": 7.4002169151715815,
+      "learning_rate": 9.846336591393832e-07,
+      "logits/chosen": -0.03831852972507477,
+      "logits/rejected": 0.09143869578838348,
+      "logps/chosen": -1.2990386486053467,
+      "logps/rejected": -1.3768006563186646,
+      "loss": 1.6703,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.2990386486053467,
+      "rewards/margins": 0.07776188850402832,
+      "rewards/rejected": -1.3768006563186646,
+      "step": 960
+    },
+    {
+      "epoch": 0.5164743268105034,
+      "grad_norm": 7.213188120271872,
+      "learning_rate": 9.842481723427704e-07,
+      "logits/chosen": 0.026158872991800308,
+      "logits/rejected": 0.0104296263307333,
+      "logps/chosen": -1.3406366109848022,
+      "logps/rejected": -1.4973324537277222,
+      "loss": 1.6826,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.3406366109848022,
+      "rewards/margins": 0.1566958725452423,
+      "rewards/rejected": -1.4973324537277222,
+      "step": 965
+    },
+    {
+      "epoch": 0.519150359591905,
+      "grad_norm": 13.922579926966854,
+      "learning_rate": 9.838579873682658e-07,
+      "logits/chosen": 0.04408761113882065,
+      "logits/rejected": 0.041850827634334564,
+      "logps/chosen": -1.223388910293579,
+      "logps/rejected": -1.309316873550415,
+      "loss": 1.6618,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.223388910293579,
+      "rewards/margins": 0.08592810481786728,
+      "rewards/rejected": -1.309316873550415,
+      "step": 970
+    },
+    {
+      "epoch": 0.5218263923733065,
+      "grad_norm": 7.122156841654528,
+      "learning_rate": 9.834631080014457e-07,
+      "logits/chosen": -0.09621085971593857,
+      "logits/rejected": 0.05997641757130623,
+      "logps/chosen": -1.2935442924499512,
+      "logps/rejected": -1.385911226272583,
+      "loss": 1.6683,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.2935442924499512,
+      "rewards/margins": 0.09236685931682587,
+      "rewards/rejected": -1.385911226272583,
+      "step": 975
+    },
+    {
+      "epoch": 0.5245024251547081,
+      "grad_norm": 12.369058104273913,
+      "learning_rate": 9.830635380734312e-07,
+      "logits/chosen": -0.09873241931200027,
+      "logits/rejected": 0.07674108445644379,
+      "logps/chosen": -1.3272755146026611,
+      "logps/rejected": -1.425257921218872,
+      "loss": 1.7112,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.3272755146026611,
+      "rewards/margins": 0.09798254072666168,
+      "rewards/rejected": -1.425257921218872,
+      "step": 980
+    },
+    {
+      "epoch": 0.5271784579361097,
+      "grad_norm": 9.248028376973298,
+      "learning_rate": 9.826592814608517e-07,
+      "logits/chosen": 0.0055808136239647865,
+      "logits/rejected": 0.17035320401191711,
+      "logps/chosen": -1.3002865314483643,
+      "logps/rejected": -1.4181766510009766,
+      "loss": 1.6661,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.3002865314483643,
+      "rewards/margins": 0.11789016425609589,
+      "rewards/rejected": -1.4181766510009766,
+      "step": 985
+    },
+    {
+      "epoch": 0.5298544907175113,
+      "grad_norm": 7.1786141635335765,
+      "learning_rate": 9.822503420858067e-07,
+      "logits/chosen": 0.04005901888012886,
+      "logits/rejected": 0.07813599705696106,
+      "logps/chosen": -1.1756422519683838,
+      "logps/rejected": -1.385040283203125,
+      "loss": 1.5564,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.1756422519683838,
+      "rewards/margins": 0.2093980312347412,
+      "rewards/rejected": -1.385040283203125,
+      "step": 990
+    },
+    {
+      "epoch": 0.5325305234989128,
+      "grad_norm": 7.924804989519409,
+      "learning_rate": 9.818367239158277e-07,
+      "logits/chosen": 0.059029437601566315,
+      "logits/rejected": 0.11863996833562851,
+      "logps/chosen": -1.2930384874343872,
+      "logps/rejected": -1.311851978302002,
+      "loss": 1.7088,
+      "rewards/accuracies": 0.45625001192092896,
+      "rewards/chosen": -1.2930384874343872,
+      "rewards/margins": 0.018813494592905045,
+      "rewards/rejected": -1.311851978302002,
+      "step": 995
+    },
+    {
+      "epoch": 0.5352065562803144,
+      "grad_norm": 7.403299342167094,
+      "learning_rate": 9.8141843096384e-07,
+      "logits/chosen": 0.05010901764035225,
+      "logits/rejected": 0.1410784125328064,
+      "logps/chosen": -1.3167353868484497,
+      "logps/rejected": -1.435499668121338,
+      "loss": 1.6718,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.3167353868484497,
+      "rewards/margins": 0.11876416206359863,
+      "rewards/rejected": -1.435499668121338,
+      "step": 1000
+    },
+    {
+      "epoch": 0.537882589061716,
+      "grad_norm": 9.467155946645837,
+      "learning_rate": 9.809954672881237e-07,
+      "logits/chosen": 0.011445349082350731,
+      "logits/rejected": 0.1570497453212738,
+      "logps/chosen": -1.3170721530914307,
+      "logps/rejected": -1.3973153829574585,
+      "loss": 1.7062,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.3170721530914307,
+      "rewards/margins": 0.08024324476718903,
+      "rewards/rejected": -1.3973153829574585,
+      "step": 1005
+    },
+    {
+      "epoch": 0.5405586218431175,
+      "grad_norm": 6.196994007674593,
+      "learning_rate": 9.80567836992274e-07,
+      "logits/chosen": -0.00795581191778183,
+      "logits/rejected": 0.14944200217723846,
+      "logps/chosen": -1.1793162822723389,
+      "logps/rejected": -1.3786524534225464,
+      "loss": 1.5712,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.1793162822723389,
+      "rewards/margins": 0.19933614134788513,
+      "rewards/rejected": -1.3786524534225464,
+      "step": 1010
+    },
+    {
+      "epoch": 0.5432346546245191,
+      "grad_norm": 8.822340000877439,
+      "learning_rate": 9.801355442251625e-07,
+      "logits/chosen": -0.002402575220912695,
+      "logits/rejected": 0.1537180095911026,
+      "logps/chosen": -1.2487362623214722,
+      "logps/rejected": -1.4060825109481812,
+      "loss": 1.6326,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.2487362623214722,
+      "rewards/margins": 0.15734639763832092,
+      "rewards/rejected": -1.4060825109481812,
+      "step": 1015
+    },
+    {
+      "epoch": 0.5459106874059207,
+      "grad_norm": 8.590755779674463,
+      "learning_rate": 9.796985931808949e-07,
+      "logits/chosen": 0.014068256132304668,
+      "logits/rejected": 0.14183005690574646,
+      "logps/chosen": -1.3020049333572388,
+      "logps/rejected": -1.4582527875900269,
+      "loss": 1.6691,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.3020049333572388,
+      "rewards/margins": 0.15624786913394928,
+      "rewards/rejected": -1.4582527875900269,
+      "step": 1020
+    },
+    {
+      "epoch": 0.5485867201873222,
+      "grad_norm": 7.752168879995463,
+      "learning_rate": 9.792569880987724e-07,
+      "logits/chosen": -0.04073721542954445,
+      "logits/rejected": 0.060070596635341644,
+      "logps/chosen": -1.203337550163269,
+      "logps/rejected": -1.4392364025115967,
+      "loss": 1.5607,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.203337550163269,
+      "rewards/margins": 0.23589889705181122,
+      "rewards/rejected": -1.4392364025115967,
+      "step": 1025
+    },
+    {
+      "epoch": 0.5512627529687238,
+      "grad_norm": 8.062073517399023,
+      "learning_rate": 9.788107332632493e-07,
+      "logits/chosen": -0.02512373961508274,
+      "logits/rejected": 0.05384986475110054,
+      "logps/chosen": -1.2794196605682373,
+      "logps/rejected": -1.3432122468948364,
+      "loss": 1.6633,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.2794196605682373,
+      "rewards/margins": 0.06379257887601852,
+      "rewards/rejected": -1.3432122468948364,
+      "step": 1030
+    },
+    {
+      "epoch": 0.5539387857501255,
+      "grad_norm": 7.008867071946321,
+      "learning_rate": 9.783598330038924e-07,
+      "logits/chosen": -0.035729292780160904,
+      "logits/rejected": 0.06643488258123398,
+      "logps/chosen": -1.3670885562896729,
+      "logps/rejected": -1.400439977645874,
+      "loss": 1.7529,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.3670885562896729,
+      "rewards/margins": 0.033351391553878784,
+      "rewards/rejected": -1.400439977645874,
+      "step": 1035
+    },
+    {
+      "epoch": 0.5566148185315271,
+      "grad_norm": 7.184521705306499,
+      "learning_rate": 9.779042916953376e-07,
+      "logits/chosen": -0.01730963960289955,
+      "logits/rejected": 0.10935752093791962,
+      "logps/chosen": -1.2887794971466064,
+      "logps/rejected": -1.4097373485565186,
+      "loss": 1.6628,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.2887794971466064,
+      "rewards/margins": 0.12095792591571808,
+      "rewards/rejected": -1.4097373485565186,
+      "step": 1040
+    },
+    {
+      "epoch": 0.5592908513129285,
+      "grad_norm": 6.354357386659293,
+      "learning_rate": 9.774441137572487e-07,
+      "logits/chosen": -0.07398457825183868,
+      "logits/rejected": 0.04312765598297119,
+      "logps/chosen": -1.2651703357696533,
+      "logps/rejected": -1.4347505569458008,
+      "loss": 1.6325,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.2651703357696533,
+      "rewards/margins": 0.16958020627498627,
+      "rewards/rejected": -1.4347505569458008,
+      "step": 1045
+    },
+    {
+      "epoch": 0.5619668840943302,
+      "grad_norm": 12.460384564896605,
+      "learning_rate": 9.76979303654274e-07,
+      "logits/chosen": -0.0898417979478836,
+      "logits/rejected": -0.005552306771278381,
+      "logps/chosen": -1.308148980140686,
+      "logps/rejected": -1.4421528577804565,
+      "loss": 1.6773,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -1.308148980140686,
+      "rewards/margins": 0.1340039223432541,
+      "rewards/rejected": -1.4421528577804565,
+      "step": 1050
+    },
+    {
+      "epoch": 0.5646429168757318,
+      "grad_norm": 11.386979354888513,
+      "learning_rate": 9.765098658960035e-07,
+      "logits/chosen": -0.015278068371117115,
+      "logits/rejected": 0.05204174667596817,
+      "logps/chosen": -1.2988243103027344,
+      "logps/rejected": -1.4415223598480225,
+      "loss": 1.6623,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.2988243103027344,
+      "rewards/margins": 0.14269819855690002,
+      "rewards/rejected": -1.4415223598480225,
+      "step": 1055
+    },
+    {
+      "epoch": 0.5673189496571333,
+      "grad_norm": 8.994374756793784,
+      "learning_rate": 9.76035805036924e-07,
+      "logits/chosen": 0.011766267940402031,
+      "logits/rejected": 0.16604655981063843,
+      "logps/chosen": -1.367139458656311,
+      "logps/rejected": -1.4718599319458008,
+      "loss": 1.7199,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.367139458656311,
+      "rewards/margins": 0.10472049564123154,
+      "rewards/rejected": -1.4718599319458008,
+      "step": 1060
+    },
+    {
+      "epoch": 0.5699949824385349,
+      "grad_norm": 5.90780125638111,
+      "learning_rate": 9.755571256763764e-07,
+      "logits/chosen": 0.04043951630592346,
+      "logits/rejected": 0.15448689460754395,
+      "logps/chosen": -1.2304548025131226,
+      "logps/rejected": -1.4319727420806885,
+      "loss": 1.5995,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.2304548025131226,
+      "rewards/margins": 0.20151777565479279,
+      "rewards/rejected": -1.4319727420806885,
+      "step": 1065
+    },
+    {
+      "epoch": 0.5726710152199365,
+      "grad_norm": 7.0689647235553394,
+      "learning_rate": 9.750738324585097e-07,
+      "logits/chosen": -0.09330417960882187,
+      "logits/rejected": 0.12764659523963928,
+      "logps/chosen": -1.286632776260376,
+      "logps/rejected": -1.462348461151123,
+      "loss": 1.6161,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.286632776260376,
+      "rewards/margins": 0.1757156103849411,
+      "rewards/rejected": -1.462348461151123,
+      "step": 1070
+    },
+    {
+      "epoch": 0.5753470480013381,
+      "grad_norm": 6.9557637626714435,
+      "learning_rate": 9.74585930072237e-07,
+      "logits/chosen": -0.008208471350371838,
+      "logits/rejected": 0.10426203161478043,
+      "logps/chosen": -1.2359673976898193,
+      "logps/rejected": -1.4388471841812134,
+      "loss": 1.6272,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.2359673976898193,
+      "rewards/margins": 0.20287981629371643,
+      "rewards/rejected": -1.4388471841812134,
+      "step": 1075
+    },
+    {
+      "epoch": 0.5780230807827396,
+      "grad_norm": 7.520732608296653,
+      "learning_rate": 9.740934232511892e-07,
+      "logits/chosen": -0.08178670704364777,
+      "logits/rejected": 0.015749135985970497,
+      "logps/chosen": -1.3555552959442139,
+      "logps/rejected": -1.3882399797439575,
+      "loss": 1.7303,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -1.3555552959442139,
+      "rewards/margins": 0.03268469125032425,
+      "rewards/rejected": -1.3882399797439575,
+      "step": 1080
+    },
+    {
+      "epoch": 0.5806991135641412,
+      "grad_norm": 6.986884533733415,
+      "learning_rate": 9.735963167736698e-07,
+      "logits/chosen": -0.022600090131163597,
+      "logits/rejected": 0.12094497680664062,
+      "logps/chosen": -1.3131974935531616,
+      "logps/rejected": -1.345354676246643,
+      "loss": 1.7244,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -1.3131974935531616,
+      "rewards/margins": 0.032157205045223236,
+      "rewards/rejected": -1.345354676246643,
+      "step": 1085
+    },
+    {
+      "epoch": 0.5833751463455428,
+      "grad_norm": 5.930443648116194,
+      "learning_rate": 9.730946154626078e-07,
+      "logits/chosen": -0.00549353426322341,
+      "logits/rejected": 0.08679809421300888,
+      "logps/chosen": -1.2895848751068115,
+      "logps/rejected": -1.3172898292541504,
+      "loss": 1.699,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -1.2895848751068115,
+      "rewards/margins": 0.027705058455467224,
+      "rewards/rejected": -1.3172898292541504,
+      "step": 1090
+    },
+    {
+      "epoch": 0.5860511791269443,
+      "grad_norm": 9.926583525201936,
+      "learning_rate": 9.725883241855117e-07,
+      "logits/chosen": -0.14290958642959595,
+      "logits/rejected": -0.02356121316552162,
+      "logps/chosen": -1.2431046962738037,
+      "logps/rejected": -1.3776899576187134,
+      "loss": 1.6306,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.2431046962738037,
+      "rewards/margins": 0.1345853954553604,
+      "rewards/rejected": -1.3776899576187134,
+      "step": 1095
+    },
+    {
+      "epoch": 0.5887272119083459,
+      "grad_norm": 8.687027405990982,
+      "learning_rate": 9.720774478544218e-07,
+      "logits/chosen": -0.004207250662147999,
+      "logits/rejected": 0.08541065454483032,
+      "logps/chosen": -1.1548147201538086,
+      "logps/rejected": -1.4498443603515625,
+      "loss": 1.5286,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.1548147201538086,
+      "rewards/margins": 0.295029878616333,
+      "rewards/rejected": -1.4498443603515625,
+      "step": 1100
+    },
+    {
+      "epoch": 0.5914032446897475,
+      "grad_norm": 6.933596671987193,
+      "learning_rate": 9.715619914258624e-07,
+      "logits/chosen": -0.05270897224545479,
+      "logits/rejected": 0.010835884138941765,
+      "logps/chosen": -1.2801262140274048,
+      "logps/rejected": -1.3806558847427368,
+      "loss": 1.6644,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.2801262140274048,
+      "rewards/margins": 0.10052965581417084,
+      "rewards/rejected": -1.3806558847427368,
+      "step": 1105
+    },
+    {
+      "epoch": 0.594079277471149,
+      "grad_norm": 8.646665805004428,
+      "learning_rate": 9.710419599007937e-07,
+      "logits/chosen": -0.04489428550004959,
+      "logits/rejected": 0.06177765130996704,
+      "logps/chosen": -1.26051926612854,
+      "logps/rejected": -1.2954927682876587,
+      "loss": 1.6631,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -1.26051926612854,
+      "rewards/margins": 0.03497341275215149,
+      "rewards/rejected": -1.2954927682876587,
+      "step": 1110
+    },
+    {
+      "epoch": 0.5967553102525506,
+      "grad_norm": 12.184135617650634,
+      "learning_rate": 9.705173583245643e-07,
+      "logits/chosen": 0.06277371942996979,
+      "logits/rejected": 0.1672242432832718,
+      "logps/chosen": -1.184098482131958,
+      "logps/rejected": -1.371530532836914,
+      "loss": 1.5668,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.184098482131958,
+      "rewards/margins": 0.18743225932121277,
+      "rewards/rejected": -1.371530532836914,
+      "step": 1115
+    },
+    {
+      "epoch": 0.5994313430339522,
+      "grad_norm": 8.431296788619152,
+      "learning_rate": 9.699881917868609e-07,
+      "logits/chosen": -0.13257569074630737,
+      "logits/rejected": -0.0393402986228466,
+      "logps/chosen": -1.2535715103149414,
+      "logps/rejected": -1.3950719833374023,
+      "loss": 1.6096,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.2535715103149414,
+      "rewards/margins": 0.1415005922317505,
+      "rewards/rejected": -1.3950719833374023,
+      "step": 1120
+    },
+    {
+      "epoch": 0.6021073758153538,
+      "grad_norm": 9.393854795053596,
+      "learning_rate": 9.694544654216594e-07,
+      "logits/chosen": -0.13203957676887512,
+      "logits/rejected": 0.030075013637542725,
+      "logps/chosen": -1.2522079944610596,
+      "logps/rejected": -1.3912581205368042,
+      "loss": 1.63,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.2522079944610596,
+      "rewards/margins": 0.1390499621629715,
+      "rewards/rejected": -1.3912581205368042,
+      "step": 1125
+    },
+    {
+      "epoch": 0.6047834085967553,
+      "grad_norm": 7.378137478880466,
+      "learning_rate": 9.689161844071755e-07,
+      "logits/chosen": 0.03407396003603935,
+      "logits/rejected": 0.08551694452762604,
+      "logps/chosen": -1.293287992477417,
+      "logps/rejected": -1.435396432876587,
+      "loss": 1.6505,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.293287992477417,
+      "rewards/margins": 0.14210841059684753,
+      "rewards/rejected": -1.435396432876587,
+      "step": 1130
+    },
+    {
+      "epoch": 0.6074594413781569,
+      "grad_norm": 10.528202132639327,
+      "learning_rate": 9.683733539658138e-07,
+      "logits/chosen": -0.03661159425973892,
+      "logits/rejected": 0.10714194923639297,
+      "logps/chosen": -1.3035916090011597,
+      "logps/rejected": -1.490254282951355,
+      "loss": 1.6403,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.3035916090011597,
+      "rewards/margins": 0.18666283786296844,
+      "rewards/rejected": -1.490254282951355,
+      "step": 1135
+    },
+    {
+      "epoch": 0.6101354741595585,
+      "grad_norm": 7.904780641376041,
+      "learning_rate": 9.678259793641178e-07,
+      "logits/chosen": -0.03270817548036575,
+      "logits/rejected": -0.005903895013034344,
+      "logps/chosen": -1.281888723373413,
+      "logps/rejected": -1.306801199913025,
+      "loss": 1.6899,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -1.281888723373413,
+      "rewards/margins": 0.024912385269999504,
+      "rewards/rejected": -1.306801199913025,
+      "step": 1140
+    },
+    {
+      "epoch": 0.61281150694096,
+      "grad_norm": 7.430297222491973,
+      "learning_rate": 9.672740659127183e-07,
+      "logits/chosen": -0.15106946229934692,
+      "logits/rejected": -0.05194350332021713,
+      "logps/chosen": -1.3074098825454712,
+      "logps/rejected": -1.4172364473342896,
+      "loss": 1.6766,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.3074098825454712,
+      "rewards/margins": 0.10982663929462433,
+      "rewards/rejected": -1.4172364473342896,
+      "step": 1145
+    },
+    {
+      "epoch": 0.6154875397223616,
+      "grad_norm": 8.04457179946613,
+      "learning_rate": 9.667176189662818e-07,
+      "logits/chosen": -0.14218200743198395,
+      "logits/rejected": -0.017552126199007034,
+      "logps/chosen": -1.1962858438491821,
+      "logps/rejected": -1.341317892074585,
+      "loss": 1.5791,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.1962858438491821,
+      "rewards/margins": 0.14503201842308044,
+      "rewards/rejected": -1.341317892074585,
+      "step": 1150
+    },
+    {
+      "epoch": 0.6181635725037632,
+      "grad_norm": 6.891796025143772,
+      "learning_rate": 9.661566439234592e-07,
+      "logits/chosen": -0.0063630566000938416,
+      "logits/rejected": 0.06813793629407883,
+      "logps/chosen": -1.2960070371627808,
+      "logps/rejected": -1.3874499797821045,
+      "loss": 1.6781,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.2960070371627808,
+      "rewards/margins": 0.09144283831119537,
+      "rewards/rejected": -1.3874499797821045,
+      "step": 1155
+    },
+    {
+      "epoch": 0.6208396052851648,
+      "grad_norm": 7.694183281262976,
+      "learning_rate": 9.655911462268327e-07,
+      "logits/chosen": 0.039759453386068344,
+      "logits/rejected": 0.11061426252126694,
+      "logps/chosen": -1.2482908964157104,
+      "logps/rejected": -1.3631278276443481,
+      "loss": 1.619,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.2482908964157104,
+      "rewards/margins": 0.11483697593212128,
+      "rewards/rejected": -1.3631278276443481,
+      "step": 1160
+    },
+    {
+      "epoch": 0.6235156380665663,
+      "grad_norm": 5.952230367438864,
+      "learning_rate": 9.650211313628636e-07,
+      "logits/chosen": -0.033538371324539185,
+      "logits/rejected": 0.03427625447511673,
+      "logps/chosen": -1.1877871751785278,
+      "logps/rejected": -1.3554905652999878,
+      "loss": 1.5718,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.1877871751785278,
+      "rewards/margins": 0.16770341992378235,
+      "rewards/rejected": -1.3554905652999878,
+      "step": 1165
+    },
+    {
+      "epoch": 0.6261916708479679,
+      "grad_norm": 5.896613098675456,
+      "learning_rate": 9.644466048618386e-07,
+      "logits/chosen": -0.07759127020835876,
+      "logits/rejected": 0.0643915981054306,
+      "logps/chosen": -1.4012242555618286,
+      "logps/rejected": -1.4206397533416748,
+      "loss": 1.7616,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.4012242555618286,
+      "rewards/margins": 0.0194155965000391,
+      "rewards/rejected": -1.4206397533416748,
+      "step": 1170
+    },
+    {
+      "epoch": 0.6288677036293695,
+      "grad_norm": 6.190941374554223,
+      "learning_rate": 9.63867572297816e-07,
+      "logits/chosen": -0.03675876185297966,
+      "logits/rejected": 0.1220620647072792,
+      "logps/chosen": -1.2355022430419922,
+      "logps/rejected": -1.3272840976715088,
+      "loss": 1.6463,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.2355022430419922,
+      "rewards/margins": 0.09178170561790466,
+      "rewards/rejected": -1.3272840976715088,
+      "step": 1175
+    },
+    {
+      "epoch": 0.631543736410771,
+      "grad_norm": 5.84549471884131,
+      "learning_rate": 9.632840392885727e-07,
+      "logits/chosen": -0.04186251014471054,
+      "logits/rejected": 0.071475088596344,
+      "logps/chosen": -1.336317539215088,
+      "logps/rejected": -1.5061604976654053,
+      "loss": 1.6709,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.336317539215088,
+      "rewards/margins": 0.16984297335147858,
+      "rewards/rejected": -1.5061604976654053,
+      "step": 1180
+    },
+    {
+      "epoch": 0.6342197691921726,
+      "grad_norm": 9.110628218510094,
+      "learning_rate": 9.626960114955483e-07,
+      "logits/chosen": 0.004259307868778706,
+      "logits/rejected": 0.11975675821304321,
+      "logps/chosen": -1.3721723556518555,
+      "logps/rejected": -1.4441169500350952,
+      "loss": 1.7389,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -1.3721723556518555,
+      "rewards/margins": 0.07194463908672333,
+      "rewards/rejected": -1.4441169500350952,
+      "step": 1185
+    },
+    {
+      "epoch": 0.6368958019735742,
+      "grad_norm": 8.604588520350909,
+      "learning_rate": 9.621034946237909e-07,
+      "logits/chosen": -0.06812702864408493,
+      "logits/rejected": 0.053222037851810455,
+      "logps/chosen": -1.32889723777771,
+      "logps/rejected": -1.4761818647384644,
+      "loss": 1.6724,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.32889723777771,
+      "rewards/margins": 0.147284597158432,
+      "rewards/rejected": -1.4761818647384644,
+      "step": 1190
+    },
+    {
+      "epoch": 0.6395718347549757,
+      "grad_norm": 9.709864140064102,
+      "learning_rate": 9.615064944219021e-07,
+      "logits/chosen": -0.0022974193561822176,
+      "logits/rejected": 0.10469532012939453,
+      "logps/chosen": -1.2060132026672363,
+      "logps/rejected": -1.4014453887939453,
+      "loss": 1.5608,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.2060132026672363,
+      "rewards/margins": 0.19543221592903137,
+      "rewards/rejected": -1.4014453887939453,
+      "step": 1195
+    },
+    {
+      "epoch": 0.6422478675363773,
+      "grad_norm": 7.482539490205642,
+      "learning_rate": 9.609050166819803e-07,
+      "logits/chosen": -0.06645162403583527,
+      "logits/rejected": -0.013482004404067993,
+      "logps/chosen": -1.2427408695220947,
+      "logps/rejected": -1.3890235424041748,
+      "loss": 1.6204,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.2427408695220947,
+      "rewards/margins": 0.1462826430797577,
+      "rewards/rejected": -1.3890235424041748,
+      "step": 1200
+    },
+    {
+      "epoch": 0.6422478675363773,
+      "eval_logits/chosen": 0.27460145950317383,
+      "eval_logits/rejected": 0.3559272885322571,
+      "eval_logps/chosen": -1.308514952659607,
+      "eval_logps/rejected": -1.4525140523910522,
+      "eval_loss": 1.6640385389328003,
+      "eval_rewards/accuracies": 0.5556379556655884,
+      "eval_rewards/chosen": -1.308514952659607,
+      "eval_rewards/margins": 0.14399926364421844,
+      "eval_rewards/rejected": -1.4525140523910522,
+      "eval_runtime": 40.6531,
+      "eval_samples_per_second": 33.085,
+      "eval_steps_per_second": 8.29,
+      "step": 1200
+    },
+    {
+      "epoch": 0.6449239003177789,
+      "grad_norm": 10.047042764127452,
+      "learning_rate": 9.602990672395653e-07,
+      "logits/chosen": -0.13160620629787445,
+      "logits/rejected": 0.02522926963865757,
+      "logps/chosen": -1.2614212036132812,
+      "logps/rejected": -1.3818312883377075,
+      "loss": 1.6262,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.2614212036132812,
+      "rewards/margins": 0.12041006237268448,
+      "rewards/rejected": -1.3818312883377075,
+      "step": 1205
+    },
+    {
+      "epoch": 0.6475999330991805,
+      "grad_norm": 9.75320853357286,
+      "learning_rate": 9.59688651973581e-07,
+      "logits/chosen": -0.06755046546459198,
+      "logits/rejected": 0.10317499935626984,
+      "logps/chosen": -1.298020362854004,
+      "logps/rejected": -1.405084490776062,
+      "loss": 1.6968,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.298020362854004,
+      "rewards/margins": 0.1070641279220581,
+      "rewards/rejected": -1.405084490776062,
+      "step": 1210
+    },
+    {
+      "epoch": 0.650275965880582,
+      "grad_norm": 6.399735782427771,
+      "learning_rate": 9.590737768062792e-07,
+      "logits/chosen": -0.12315372377634048,
+      "logits/rejected": -0.013821613974869251,
+      "logps/chosen": -1.2987453937530518,
+      "logps/rejected": -1.325515866279602,
+      "loss": 1.6791,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.2987453937530518,
+      "rewards/margins": 0.026770705357193947,
+      "rewards/rejected": -1.325515866279602,
+      "step": 1215
+    },
+    {
+      "epoch": 0.6529519986619836,
+      "grad_norm": 8.322102796728917,
+      "learning_rate": 9.584544477031816e-07,
+      "logits/chosen": 0.055944450199604034,
+      "logits/rejected": 0.15044358372688293,
+      "logps/chosen": -1.2174725532531738,
+      "logps/rejected": -1.3485443592071533,
+      "loss": 1.6243,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.2174725532531738,
+      "rewards/margins": 0.13107183575630188,
+      "rewards/rejected": -1.3485443592071533,
+      "step": 1220
+    },
+    {
+      "epoch": 0.6556280314433852,
+      "grad_norm": 5.972083585488595,
+      "learning_rate": 9.578306706730215e-07,
+      "logits/chosen": -0.15188264846801758,
+      "logits/rejected": 0.03867144137620926,
+      "logps/chosen": -1.3097045421600342,
+      "logps/rejected": -1.4055533409118652,
+      "loss": 1.6674,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.3097045421600342,
+      "rewards/margins": 0.09584885090589523,
+      "rewards/rejected": -1.4055533409118652,
+      "step": 1225
+    },
+    {
+      "epoch": 0.6583040642247867,
+      "grad_norm": 8.093006246489878,
+      "learning_rate": 9.572024517676865e-07,
+      "logits/chosen": -0.0809139758348465,
+      "logits/rejected": 0.012979112565517426,
+      "logps/chosen": -1.2431614398956299,
+      "logps/rejected": -1.4049303531646729,
+      "loss": 1.6231,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.2431614398956299,
+      "rewards/margins": 0.1617688238620758,
+      "rewards/rejected": -1.4049303531646729,
+      "step": 1230
+    },
+    {
+      "epoch": 0.6609800970061883,
+      "grad_norm": 6.848159379031791,
+      "learning_rate": 9.565697970821593e-07,
+      "logits/chosen": -0.050653569400310516,
+      "logits/rejected": 0.06407668441534042,
+      "logps/chosen": -1.2996481657028198,
+      "logps/rejected": -1.378269910812378,
+      "loss": 1.6866,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.2996481657028198,
+      "rewards/margins": 0.07862164080142975,
+      "rewards/rejected": -1.378269910812378,
+      "step": 1235
+    },
+    {
+      "epoch": 0.6636561297875899,
+      "grad_norm": 8.510686496424805,
+      "learning_rate": 9.559327127544585e-07,
+      "logits/chosen": -0.16195550560951233,
+      "logits/rejected": -0.04807741194963455,
+      "logps/chosen": -1.269803762435913,
+      "logps/rejected": -1.3921595811843872,
+      "loss": 1.6484,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.269803762435913,
+      "rewards/margins": 0.12235573679208755,
+      "rewards/rejected": -1.3921595811843872,
+      "step": 1240
+    },
+    {
+      "epoch": 0.6663321625689914,
+      "grad_norm": 7.838294023870456,
+      "learning_rate": 9.552912049655789e-07,
+      "logits/chosen": -0.07872875034809113,
+      "logits/rejected": 0.07285197079181671,
+      "logps/chosen": -1.3556663990020752,
+      "logps/rejected": -1.3947060108184814,
+      "loss": 1.713,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -1.3556663990020752,
+      "rewards/margins": 0.039039671421051025,
+      "rewards/rejected": -1.3947060108184814,
+      "step": 1245
+    },
+    {
+      "epoch": 0.669008195350393,
+      "grad_norm": 11.117903128068692,
+      "learning_rate": 9.546452799394315e-07,
+      "logits/chosen": -0.06848958134651184,
+      "logits/rejected": 0.09921123087406158,
+      "logps/chosen": -1.3285969495773315,
+      "logps/rejected": -1.3818318843841553,
+      "loss": 1.7038,
+      "rewards/accuracies": 0.45625001192092896,
+      "rewards/chosen": -1.3285969495773315,
+      "rewards/margins": 0.05323495343327522,
+      "rewards/rejected": -1.3818318843841553,
+      "step": 1250
+    },
+    {
+      "epoch": 0.6716842281317946,
+      "grad_norm": 9.426299726196365,
+      "learning_rate": 9.539949439427846e-07,
+      "logits/chosen": -0.10870027542114258,
+      "logits/rejected": 0.004453069064766169,
+      "logps/chosen": -1.2898695468902588,
+      "logps/rejected": -1.387575387954712,
+      "loss": 1.6565,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.2898695468902588,
+      "rewards/margins": 0.0977058857679367,
+      "rewards/rejected": -1.387575387954712,
+      "step": 1255
+    },
+    {
+      "epoch": 0.6743602609131962,
+      "grad_norm": 7.7439778835708255,
+      "learning_rate": 9.533402032852002e-07,
+      "logits/chosen": -0.12049994617700577,
+      "logits/rejected": -0.005878242664039135,
+      "logps/chosen": -1.2310149669647217,
+      "logps/rejected": -1.411268711090088,
+      "loss": 1.6042,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.2310149669647217,
+      "rewards/margins": 0.18025359511375427,
+      "rewards/rejected": -1.411268711090088,
+      "step": 1260
+    },
+    {
+      "epoch": 0.6770362936945977,
+      "grad_norm": 6.7439236704393535,
+      "learning_rate": 9.526810643189754e-07,
+      "logits/chosen": -0.02287997305393219,
+      "logits/rejected": 0.09953973442316055,
+      "logps/chosen": -1.2751572132110596,
+      "logps/rejected": -1.3866431713104248,
+      "loss": 1.6341,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.2751572132110596,
+      "rewards/margins": 0.11148588359355927,
+      "rewards/rejected": -1.3866431713104248,
+      "step": 1265
+    },
+    {
+      "epoch": 0.6797123264759993,
+      "grad_norm": 7.899839971390841,
+      "learning_rate": 9.52017533439079e-07,
+      "logits/chosen": -0.08607939630746841,
+      "logits/rejected": 0.0121527761220932,
+      "logps/chosen": -1.2371551990509033,
+      "logps/rejected": -1.392277717590332,
+      "loss": 1.6179,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.2371551990509033,
+      "rewards/margins": 0.15512245893478394,
+      "rewards/rejected": -1.392277717590332,
+      "step": 1270
+    },
+    {
+      "epoch": 0.6823883592574009,
+      "grad_norm": 5.628595303817242,
+      "learning_rate": 9.513496170830909e-07,
+      "logits/chosen": -0.0971599891781807,
+      "logits/rejected": -0.005085567478090525,
+      "logps/chosen": -1.2943452596664429,
+      "logps/rejected": -1.3745836019515991,
+      "loss": 1.6751,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -1.2943452596664429,
+      "rewards/margins": 0.08023835718631744,
+      "rewards/rejected": -1.3745836019515991,
+      "step": 1275
+    },
+    {
+      "epoch": 0.6850643920388024,
+      "grad_norm": 8.352724391218983,
+      "learning_rate": 9.506773217311382e-07,
+      "logits/chosen": -0.0870608538389206,
+      "logits/rejected": 0.03902773559093475,
+      "logps/chosen": -1.3659374713897705,
+      "logps/rejected": -1.4287694692611694,
+      "loss": 1.7331,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -1.3659374713897705,
+      "rewards/margins": 0.06283222138881683,
+      "rewards/rejected": -1.4287694692611694,
+      "step": 1280
+    },
+    {
+      "epoch": 0.687740424820204,
+      "grad_norm": 7.466254419479338,
+      "learning_rate": 9.500006539058334e-07,
+      "logits/chosen": -0.028805231675505638,
+      "logits/rejected": 0.08828462660312653,
+      "logps/chosen": -1.241590976715088,
+      "logps/rejected": -1.3411766290664673,
+      "loss": 1.6123,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.241590976715088,
+      "rewards/margins": 0.09958568960428238,
+      "rewards/rejected": -1.3411766290664673,
+      "step": 1285
+    },
+    {
+      "epoch": 0.6904164576016056,
+      "grad_norm": 7.996329692726276,
+      "learning_rate": 9.493196201722109e-07,
+      "logits/chosen": -0.179660826921463,
+      "logits/rejected": -0.055936299264431,
+      "logps/chosen": -1.310693621635437,
+      "logps/rejected": -1.3532706499099731,
+      "loss": 1.7178,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -1.310693621635437,
+      "rewards/margins": 0.04257718473672867,
+      "rewards/rejected": -1.3532706499099731,
+      "step": 1290
+    },
+    {
+      "epoch": 0.6930924903830072,
+      "grad_norm": 6.1239123973147676,
+      "learning_rate": 9.486342271376628e-07,
+      "logits/chosen": -0.08025787770748138,
+      "logits/rejected": -0.07135315984487534,
+      "logps/chosen": -1.2761461734771729,
+      "logps/rejected": -1.4287402629852295,
+      "loss": 1.6428,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.2761461734771729,
+      "rewards/margins": 0.15259432792663574,
+      "rewards/rejected": -1.4287402629852295,
+      "step": 1295
+    },
+    {
+      "epoch": 0.6957685231644087,
+      "grad_norm": 7.432784798775175,
+      "learning_rate": 9.479444814518755e-07,
+      "logits/chosen": -0.0681220144033432,
+      "logits/rejected": 0.14305290579795837,
+      "logps/chosen": -1.2733452320098877,
+      "logps/rejected": -1.4136388301849365,
+      "loss": 1.6306,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.2733452320098877,
+      "rewards/margins": 0.14029362797737122,
+      "rewards/rejected": -1.4136388301849365,
+      "step": 1300
+    },
+    {
+      "epoch": 0.6984445559458103,
+      "grad_norm": 7.02831430011763,
+      "learning_rate": 9.472503898067645e-07,
+      "logits/chosen": 0.015904754400253296,
+      "logits/rejected": 0.06446841359138489,
+      "logps/chosen": -1.2825438976287842,
+      "logps/rejected": -1.4184726476669312,
+      "loss": 1.6303,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.2825438976287842,
+      "rewards/margins": 0.13592875003814697,
+      "rewards/rejected": -1.4184726476669312,
+      "step": 1305
+    },
+    {
+      "epoch": 0.701120588727212,
+      "grad_norm": 7.109526609365743,
+      "learning_rate": 9.465519589364099e-07,
+      "logits/chosen": 0.019358232617378235,
+      "logits/rejected": 0.08662734180688858,
+      "logps/chosen": -1.2953920364379883,
+      "logps/rejected": -1.4193115234375,
+      "loss": 1.6836,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.2953920364379883,
+      "rewards/margins": 0.12391936779022217,
+      "rewards/rejected": -1.4193115234375,
+      "step": 1310
+    },
+    {
+      "epoch": 0.7037966215086134,
+      "grad_norm": 6.365121027589216,
+      "learning_rate": 9.458491956169914e-07,
+      "logits/chosen": -0.07012242078781128,
+      "logits/rejected": 0.09191621094942093,
+      "logps/chosen": -1.2381283044815063,
+      "logps/rejected": -1.4403287172317505,
+      "loss": 1.6023,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.2381283044815063,
+      "rewards/margins": 0.2022004872560501,
+      "rewards/rejected": -1.4403287172317505,
+      "step": 1315
+    },
+    {
+      "epoch": 0.706472654290015,
+      "grad_norm": 5.815899579642899,
+      "learning_rate": 9.451421066667215e-07,
+      "logits/chosen": -0.16489481925964355,
+      "logits/rejected": 0.016862286254763603,
+      "logps/chosen": -1.2204926013946533,
+      "logps/rejected": -1.3832234144210815,
+      "loss": 1.5897,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.2204926013946533,
+      "rewards/margins": 0.16273091733455658,
+      "rewards/rejected": -1.3832234144210815,
+      "step": 1320
+    },
+    {
+      "epoch": 0.7091486870714167,
+      "grad_norm": 11.57452564485908,
+      "learning_rate": 9.444306989457805e-07,
+      "logits/chosen": -0.0183856338262558,
+      "logits/rejected": 0.07245060801506042,
+      "logps/chosen": -1.2934316396713257,
+      "logps/rejected": -1.3811691999435425,
+      "loss": 1.6827,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.2934316396713257,
+      "rewards/margins": 0.08773749321699142,
+      "rewards/rejected": -1.3811691999435425,
+      "step": 1325
+    },
+    {
+      "epoch": 0.7118247198528181,
+      "grad_norm": 5.568127237800386,
+      "learning_rate": 9.437149793562489e-07,
+      "logits/chosen": -0.05760542303323746,
+      "logits/rejected": 0.04698445647954941,
+      "logps/chosen": -1.2826694250106812,
+      "logps/rejected": -1.3580313920974731,
+      "loss": 1.6626,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.2826694250106812,
+      "rewards/margins": 0.07536178827285767,
+      "rewards/rejected": -1.3580313920974731,
+      "step": 1330
+    },
+    {
+      "epoch": 0.7145007526342197,
+      "grad_norm": 6.356676163069989,
+      "learning_rate": 9.429949548420417e-07,
+      "logits/chosen": -0.03230949491262436,
+      "logits/rejected": 0.036061473190784454,
+      "logps/chosen": -1.353149652481079,
+      "logps/rejected": -1.481315016746521,
+      "loss": 1.6798,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.353149652481079,
+      "rewards/margins": 0.12816523015499115,
+      "rewards/rejected": -1.481315016746521,
+      "step": 1335
+    },
+    {
+      "epoch": 0.7171767854156214,
+      "grad_norm": 7.862975720515515,
+      "learning_rate": 9.422706323888396e-07,
+      "logits/chosen": -0.02949601784348488,
+      "logits/rejected": -0.0037048873491585255,
+      "logps/chosen": -1.3224132061004639,
+      "logps/rejected": -1.4355237483978271,
+      "loss": 1.6931,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.3224132061004639,
+      "rewards/margins": 0.11311064660549164,
+      "rewards/rejected": -1.4355237483978271,
+      "step": 1340
+    },
+    {
+      "epoch": 0.719852818197023,
+      "grad_norm": 5.176058235706188,
+      "learning_rate": 9.415420190240225e-07,
+      "logits/chosen": 0.0031900019384920597,
+      "logits/rejected": 0.15508271753787994,
+      "logps/chosen": -1.3075135946273804,
+      "logps/rejected": -1.3928053379058838,
+      "loss": 1.6681,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.3075135946273804,
+      "rewards/margins": 0.0852917954325676,
+      "rewards/rejected": -1.3928053379058838,
+      "step": 1345
+    },
+    {
+      "epoch": 0.7225288509784245,
+      "grad_norm": 7.611410454276417,
+      "learning_rate": 9.408091218166002e-07,
+      "logits/chosen": 0.005640983581542969,
+      "logits/rejected": 0.05313152074813843,
+      "logps/chosen": -1.2921403646469116,
+      "logps/rejected": -1.3184449672698975,
+      "loss": 1.7019,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.2921403646469116,
+      "rewards/margins": 0.02630445919930935,
+      "rewards/rejected": -1.3184449672698975,
+      "step": 1350
+    },
+    {
+      "epoch": 0.7252048837598261,
+      "grad_norm": 7.152231165146874,
+      "learning_rate": 9.400719478771449e-07,
+      "logits/chosen": -0.010153415612876415,
+      "logits/rejected": 0.25364306569099426,
+      "logps/chosen": -1.3463757038116455,
+      "logps/rejected": -1.4059813022613525,
+      "loss": 1.727,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.3463757038116455,
+      "rewards/margins": 0.05960559844970703,
+      "rewards/rejected": -1.4059813022613525,
+      "step": 1355
+    },
+    {
+      "epoch": 0.7278809165412277,
+      "grad_norm": 8.501974366271751,
+      "learning_rate": 9.393305043577209e-07,
+      "logits/chosen": -0.14782077074050903,
+      "logits/rejected": -0.020702391862869263,
+      "logps/chosen": -1.359283685684204,
+      "logps/rejected": -1.4838364124298096,
+      "loss": 1.7063,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.359283685684204,
+      "rewards/margins": 0.1245526447892189,
+      "rewards/rejected": -1.4838364124298096,
+      "step": 1360
+    },
+    {
+      "epoch": 0.7305569493226292,
+      "grad_norm": 5.371286371507929,
+      "learning_rate": 9.38584798451817e-07,
+      "logits/chosen": -0.02379719726741314,
+      "logits/rejected": 0.09597743302583694,
+      "logps/chosen": -1.2888532876968384,
+      "logps/rejected": -1.4119470119476318,
+      "loss": 1.6511,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.2888532876968384,
+      "rewards/margins": 0.12309379875659943,
+      "rewards/rejected": -1.4119470119476318,
+      "step": 1365
+    },
+    {
+      "epoch": 0.7332329821040308,
+      "grad_norm": 8.362674846669057,
+      "learning_rate": 9.37834837394275e-07,
+      "logits/chosen": -0.04941604286432266,
+      "logits/rejected": 0.049748942255973816,
+      "logps/chosen": -1.3442224264144897,
+      "logps/rejected": -1.5253331661224365,
+      "loss": 1.6698,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.3442224264144897,
+      "rewards/margins": 0.1811104714870453,
+      "rewards/rejected": -1.5253331661224365,
+      "step": 1370
+    },
+    {
+      "epoch": 0.7359090148854324,
+      "grad_norm": 6.495126111078663,
+      "learning_rate": 9.370806284612203e-07,
+      "logits/chosen": -0.06998618692159653,
+      "logits/rejected": 0.05238911509513855,
+      "logps/chosen": -1.2650034427642822,
+      "logps/rejected": -1.4815846681594849,
+      "loss": 1.6215,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.2650034427642822,
+      "rewards/margins": 0.21658115088939667,
+      "rewards/rejected": -1.4815846681594849,
+      "step": 1375
+    },
+    {
+      "epoch": 0.738585047666834,
+      "grad_norm": 9.380202598349682,
+      "learning_rate": 9.363221789699912e-07,
+      "logits/chosen": -0.12380628287792206,
+      "logits/rejected": -0.02891998365521431,
+      "logps/chosen": -1.2920019626617432,
+      "logps/rejected": -1.347071647644043,
+      "loss": 1.695,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.2920019626617432,
+      "rewards/margins": 0.055069684982299805,
+      "rewards/rejected": -1.347071647644043,
+      "step": 1380
+    },
+    {
+      "epoch": 0.7412610804482355,
+      "grad_norm": 7.368251054782439,
+      "learning_rate": 9.355594962790682e-07,
+      "logits/chosen": -0.10633337497711182,
+      "logits/rejected": -0.004128370434045792,
+      "logps/chosen": -1.2442162036895752,
+      "logps/rejected": -1.3430384397506714,
+      "loss": 1.647,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.2442162036895752,
+      "rewards/margins": 0.09882227331399918,
+      "rewards/rejected": -1.3430384397506714,
+      "step": 1385
+    },
+    {
+      "epoch": 0.7439371132296371,
+      "grad_norm": 9.269784762432563,
+      "learning_rate": 9.34792587788002e-07,
+      "logits/chosen": 0.009227467700839043,
+      "logits/rejected": 0.10784689337015152,
+      "logps/chosen": -1.2933768033981323,
+      "logps/rejected": -1.4177485704421997,
+      "loss": 1.6632,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.2933768033981323,
+      "rewards/margins": 0.12437163293361664,
+      "rewards/rejected": -1.4177485704421997,
+      "step": 1390
+    },
+    {
+      "epoch": 0.7466131460110387,
+      "grad_norm": 6.200345019295354,
+      "learning_rate": 9.34021460937342e-07,
+      "logits/chosen": -0.014314393512904644,
+      "logits/rejected": 0.07106490433216095,
+      "logps/chosen": -1.2545225620269775,
+      "logps/rejected": -1.330452561378479,
+      "loss": 1.6528,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.2545225620269775,
+      "rewards/margins": 0.07592994719743729,
+      "rewards/rejected": -1.330452561378479,
+      "step": 1395
+    },
+    {
+      "epoch": 0.7492891787924402,
+      "grad_norm": 5.113076051211028,
+      "learning_rate": 9.332461232085646e-07,
+      "logits/chosen": -0.1846596896648407,
+      "logits/rejected": -0.076918825507164,
+      "logps/chosen": -1.3319851160049438,
+      "logps/rejected": -1.3965907096862793,
+      "loss": 1.6926,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.3319851160049438,
+      "rewards/margins": 0.06460557878017426,
+      "rewards/rejected": -1.3965907096862793,
+      "step": 1400
+    },
+    {
+      "epoch": 0.7519652115738418,
+      "grad_norm": 5.78917820670399,
+      "learning_rate": 9.324665821239998e-07,
+      "logits/chosen": -0.07831187546253204,
+      "logits/rejected": 0.07580214738845825,
+      "logps/chosen": -1.189420223236084,
+      "logps/rejected": -1.4074944257736206,
+      "loss": 1.5701,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.189420223236084,
+      "rewards/margins": 0.21807405352592468,
+      "rewards/rejected": -1.4074944257736206,
+      "step": 1405
+    },
+    {
+      "epoch": 0.7546412443552434,
+      "grad_norm": 5.6554854963582235,
+      "learning_rate": 9.316828452467583e-07,
+      "logits/chosen": -0.13492147624492645,
+      "logits/rejected": 0.018135586753487587,
+      "logps/chosen": -1.2970507144927979,
+      "logps/rejected": -1.44521164894104,
+      "loss": 1.6474,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.2970507144927979,
+      "rewards/margins": 0.14816097915172577,
+      "rewards/rejected": -1.44521164894104,
+      "step": 1410
+    },
+    {
+      "epoch": 0.7573172771366449,
+      "grad_norm": 9.906896734902677,
+      "learning_rate": 9.30894920180659e-07,
+      "logits/chosen": -0.03486012667417526,
+      "logits/rejected": 0.09014071524143219,
+      "logps/chosen": -1.3414547443389893,
+      "logps/rejected": -1.3082062005996704,
+      "loss": 1.7412,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.3414547443389893,
+      "rewards/margins": -0.033248428255319595,
+      "rewards/rejected": -1.3082062005996704,
+      "step": 1415
+    },
+    {
+      "epoch": 0.7599933099180465,
+      "grad_norm": 5.945288403773963,
+      "learning_rate": 9.301028145701543e-07,
+      "logits/chosen": -0.030293172225356102,
+      "logits/rejected": 0.07679029554128647,
+      "logps/chosen": -1.2248847484588623,
+      "logps/rejected": -1.4750934839248657,
+      "loss": 1.5933,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.2248847484588623,
+      "rewards/margins": 0.25020867586135864,
+      "rewards/rejected": -1.4750934839248657,
+      "step": 1420
+    },
+    {
+      "epoch": 0.7626693426994481,
+      "grad_norm": 6.2924481165233015,
+      "learning_rate": 9.293065361002563e-07,
+      "logits/chosen": -0.0005339503404684365,
+      "logits/rejected": 0.05334051325917244,
+      "logps/chosen": -1.2537484169006348,
+      "logps/rejected": -1.514533281326294,
+      "loss": 1.6072,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.2537484169006348,
+      "rewards/margins": 0.26078492403030396,
+      "rewards/rejected": -1.514533281326294,
+      "step": 1425
+    },
+    {
+      "epoch": 0.7653453754808497,
+      "grad_norm": 6.747990687916865,
+      "learning_rate": 9.285060924964622e-07,
+      "logits/chosen": -0.1154429093003273,
+      "logits/rejected": -0.004071956966072321,
+      "logps/chosen": -1.3178616762161255,
+      "logps/rejected": -1.3854830265045166,
+      "loss": 1.6837,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -1.3178616762161255,
+      "rewards/margins": 0.06762126833200455,
+      "rewards/rejected": -1.3854830265045166,
+      "step": 1430
+    },
+    {
+      "epoch": 0.7680214082622512,
+      "grad_norm": 7.379312272553948,
+      "learning_rate": 9.277014915246792e-07,
+      "logits/chosen": -0.001880859606899321,
+      "logits/rejected": 0.03771301731467247,
+      "logps/chosen": -1.2543996572494507,
+      "logps/rejected": -1.4419598579406738,
+      "loss": 1.6479,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.2543996572494507,
+      "rewards/margins": 0.1875600814819336,
+      "rewards/rejected": -1.4419598579406738,
+      "step": 1435
+    },
+    {
+      "epoch": 0.7706974410436528,
+      "grad_norm": 5.879818791750725,
+      "learning_rate": 9.268927409911498e-07,
+      "logits/chosen": -0.06734560430049896,
+      "logits/rejected": 0.010824119672179222,
+      "logps/chosen": -1.3159987926483154,
+      "logps/rejected": -1.3603847026824951,
+      "loss": 1.6746,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.3159987926483154,
+      "rewards/margins": 0.04438580945134163,
+      "rewards/rejected": -1.3603847026824951,
+      "step": 1440
+    },
+    {
+      "epoch": 0.7733734738250544,
+      "grad_norm": 8.30102687568028,
+      "learning_rate": 9.260798487423749e-07,
+      "logits/chosen": -0.11533383280038834,
+      "logits/rejected": 0.06859040260314941,
+      "logps/chosen": -1.3441131114959717,
+      "logps/rejected": -1.4326902627944946,
+      "loss": 1.7039,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.3441131114959717,
+      "rewards/margins": 0.08857724070549011,
+      "rewards/rejected": -1.4326902627944946,
+      "step": 1445
+    },
+    {
+      "epoch": 0.7760495066064559,
+      "grad_norm": 16.99885339856131,
+      "learning_rate": 9.252628226650389e-07,
+      "logits/chosen": -0.005267086438834667,
+      "logits/rejected": 0.0751887634396553,
+      "logps/chosen": -1.2348849773406982,
+      "logps/rejected": -1.3485267162322998,
+      "loss": 1.6351,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.2348849773406982,
+      "rewards/margins": 0.1136416420340538,
+      "rewards/rejected": -1.3485267162322998,
+      "step": 1450
+    },
+    {
+      "epoch": 0.7787255393878575,
+      "grad_norm": 9.381136737150669,
+      "learning_rate": 9.244416706859321e-07,
+      "logits/chosen": -0.05502817779779434,
+      "logits/rejected": 0.07302253693342209,
+      "logps/chosen": -1.2619531154632568,
+      "logps/rejected": -1.4499928951263428,
+      "loss": 1.6239,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.2619531154632568,
+      "rewards/margins": 0.1880396157503128,
+      "rewards/rejected": -1.4499928951263428,
+      "step": 1455
+    },
+    {
+      "epoch": 0.7814015721692591,
+      "grad_norm": 5.68119111046384,
+      "learning_rate": 9.23616400771875e-07,
+      "logits/chosen": -0.051288902759552,
+      "logits/rejected": 0.06629034131765366,
+      "logps/chosen": -1.2232128381729126,
+      "logps/rejected": -1.3965691328048706,
+      "loss": 1.5924,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.2232128381729126,
+      "rewards/margins": 0.17335614562034607,
+      "rewards/rejected": -1.3965691328048706,
+      "step": 1460
+    },
+    {
+      "epoch": 0.7840776049506607,
+      "grad_norm": 6.477388224027011,
+      "learning_rate": 9.227870209296395e-07,
+      "logits/chosen": -0.020612351596355438,
+      "logits/rejected": 0.06191522628068924,
+      "logps/chosen": -1.3425668478012085,
+      "logps/rejected": -1.4560648202896118,
+      "loss": 1.716,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.3425668478012085,
+      "rewards/margins": 0.11349793523550034,
+      "rewards/rejected": -1.4560648202896118,
+      "step": 1465
+    },
+    {
+      "epoch": 0.7867536377320622,
+      "grad_norm": 8.658190452030363,
+      "learning_rate": 9.219535392058728e-07,
+      "logits/chosen": -0.133492112159729,
+      "logits/rejected": -0.10479389131069183,
+      "logps/chosen": -1.3073797225952148,
+      "logps/rejected": -1.38153874874115,
+      "loss": 1.6891,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.3073797225952148,
+      "rewards/margins": 0.0741589292883873,
+      "rewards/rejected": -1.38153874874115,
+      "step": 1470
+    },
+    {
+      "epoch": 0.7894296705134638,
+      "grad_norm": 5.762694343210087,
+      "learning_rate": 9.211159636870181e-07,
+      "logits/chosen": -0.11618445813655853,
+      "logits/rejected": 0.019445959478616714,
+      "logps/chosen": -1.2836791276931763,
+      "logps/rejected": -1.4270191192626953,
+      "loss": 1.6231,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.2836791276931763,
+      "rewards/margins": 0.14333993196487427,
+      "rewards/rejected": -1.4270191192626953,
+      "step": 1475
+    },
+    {
+      "epoch": 0.7921057032948654,
+      "grad_norm": 7.3912636159559835,
+      "learning_rate": 9.202743024992367e-07,
+      "logits/chosen": -0.05757608264684677,
+      "logits/rejected": 0.027957985177636147,
+      "logps/chosen": -1.270872950553894,
+      "logps/rejected": -1.4571425914764404,
+      "loss": 1.6426,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.270872950553894,
+      "rewards/margins": 0.18626955151557922,
+      "rewards/rejected": -1.4571425914764404,
+      "step": 1480
+    },
+    {
+      "epoch": 0.7947817360762669,
+      "grad_norm": 7.677644725215643,
+      "learning_rate": 9.194285638083293e-07,
+      "logits/chosen": -0.013660090044140816,
+      "logits/rejected": 0.11817152798175812,
+      "logps/chosen": -1.3152987957000732,
+      "logps/rejected": -1.453066349029541,
+      "loss": 1.6646,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.3152987957000732,
+      "rewards/margins": 0.13776752352714539,
+      "rewards/rejected": -1.453066349029541,
+      "step": 1485
+    },
+    {
+      "epoch": 0.7974577688576685,
+      "grad_norm": 10.387890790192381,
+      "learning_rate": 9.185787558196562e-07,
+      "logits/chosen": -0.0556030198931694,
+      "logits/rejected": 0.036634549498558044,
+      "logps/chosen": -1.2904164791107178,
+      "logps/rejected": -1.3281852006912231,
+      "loss": 1.6845,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -1.2904164791107178,
+      "rewards/margins": 0.037768661975860596,
+      "rewards/rejected": -1.3281852006912231,
+      "step": 1490
+    },
+    {
+      "epoch": 0.8001338016390701,
+      "grad_norm": 9.354322301421194,
+      "learning_rate": 9.177248867780583e-07,
+      "logits/chosen": -0.06285317242145538,
+      "logits/rejected": 0.03233514353632927,
+      "logps/chosen": -1.3816922903060913,
+      "logps/rejected": -1.4063470363616943,
+      "loss": 1.7503,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.3816922903060913,
+      "rewards/margins": 0.024654611945152283,
+      "rewards/rejected": -1.4063470363616943,
+      "step": 1495
+    },
+    {
+      "epoch": 0.8028098344204716,
+      "grad_norm": 9.378817585928564,
+      "learning_rate": 9.168669649677769e-07,
+      "logits/chosen": -0.08374687284231186,
+      "logits/rejected": 0.006220565643161535,
+      "logps/chosen": -1.2730871438980103,
+      "logps/rejected": -1.4184569120407104,
+      "loss": 1.6471,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.2730871438980103,
+      "rewards/margins": 0.14536967873573303,
+      "rewards/rejected": -1.4184569120407104,
+      "step": 1500
+    },
+    {
+      "epoch": 0.8054858672018732,
+      "grad_norm": 9.036846103442565,
+      "learning_rate": 9.16004998712373e-07,
+      "logits/chosen": -0.0019377961289137602,
+      "logits/rejected": 0.04818776249885559,
+      "logps/chosen": -1.227207899093628,
+      "logps/rejected": -1.445753812789917,
+      "loss": 1.5801,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.227207899093628,
+      "rewards/margins": 0.21854594349861145,
+      "rewards/rejected": -1.445753812789917,
+      "step": 1505
+    },
+    {
+      "epoch": 0.8081618999832748,
+      "grad_norm": 6.099930304574893,
+      "learning_rate": 9.151389963746472e-07,
+      "logits/chosen": -0.08609070628881454,
+      "logits/rejected": 0.14547133445739746,
+      "logps/chosen": -1.322655200958252,
+      "logps/rejected": -1.4444901943206787,
+      "loss": 1.6681,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.322655200958252,
+      "rewards/margins": 0.12183494865894318,
+      "rewards/rejected": -1.4444901943206787,
+      "step": 1510
+    },
+    {
+      "epoch": 0.8108379327646764,
+      "grad_norm": 6.268135673470169,
+      "learning_rate": 9.142689663565577e-07,
+      "logits/chosen": -0.008268142119050026,
+      "logits/rejected": 0.053254224359989166,
+      "logps/chosen": -1.2595961093902588,
+      "logps/rejected": -1.3987139463424683,
+      "loss": 1.6466,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.2595961093902588,
+      "rewards/margins": 0.139118030667305,
+      "rewards/rejected": -1.3987139463424683,
+      "step": 1515
+    },
+    {
+      "epoch": 0.8135139655460779,
+      "grad_norm": 8.021966955793015,
+      "learning_rate": 9.133949170991397e-07,
+      "logits/chosen": -0.03498731926083565,
+      "logits/rejected": 0.030461037531495094,
+      "logps/chosen": -1.2982255220413208,
+      "logps/rejected": -1.4231431484222412,
+      "loss": 1.6557,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.2982255220413208,
+      "rewards/margins": 0.1249176487326622,
+      "rewards/rejected": -1.4231431484222412,
+      "step": 1520
+    },
+    {
+      "epoch": 0.8161899983274795,
+      "grad_norm": 7.341712205930606,
+      "learning_rate": 9.125168570824231e-07,
+      "logits/chosen": -0.061010561883449554,
+      "logits/rejected": 0.0892036184668541,
+      "logps/chosen": -1.2900292873382568,
+      "logps/rejected": -1.3733108043670654,
+      "loss": 1.6893,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.2900292873382568,
+      "rewards/margins": 0.08328154683113098,
+      "rewards/rejected": -1.3733108043670654,
+      "step": 1525
+    },
+    {
+      "epoch": 0.8188660311088811,
+      "grad_norm": 9.797618530298605,
+      "learning_rate": 9.116347948253496e-07,
+      "logits/chosen": -0.07041227072477341,
+      "logits/rejected": 0.012045316398143768,
+      "logps/chosen": -1.3219391107559204,
+      "logps/rejected": -1.4183783531188965,
+      "loss": 1.6705,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.3219391107559204,
+      "rewards/margins": 0.09643938392400742,
+      "rewards/rejected": -1.4183783531188965,
+      "step": 1530
+    },
+    {
+      "epoch": 0.8215420638902826,
+      "grad_norm": 9.662443664040012,
+      "learning_rate": 9.107487388856916e-07,
+      "logits/chosen": -0.10239896923303604,
+      "logits/rejected": 0.0421513170003891,
+      "logps/chosen": -1.2454922199249268,
+      "logps/rejected": -1.4122923612594604,
+      "loss": 1.6068,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.2454922199249268,
+      "rewards/margins": 0.16680024564266205,
+      "rewards/rejected": -1.4122923612594604,
+      "step": 1535
+    },
+    {
+      "epoch": 0.8242180966716842,
+      "grad_norm": 8.787552287478555,
+      "learning_rate": 9.098586978599673e-07,
+      "logits/chosen": -0.03155619651079178,
+      "logits/rejected": 0.10633011907339096,
+      "logps/chosen": -1.273524522781372,
+      "logps/rejected": -1.489218831062317,
+      "loss": 1.6086,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.273524522781372,
+      "rewards/margins": 0.2156943827867508,
+      "rewards/rejected": -1.489218831062317,
+      "step": 1540
+    },
+    {
+      "epoch": 0.8268941294530858,
+      "grad_norm": 7.029763697711535,
+      "learning_rate": 9.089646803833588e-07,
+      "logits/chosen": -0.01791234128177166,
+      "logits/rejected": 0.12071450799703598,
+      "logps/chosen": -1.2818536758422852,
+      "logps/rejected": -1.3673170804977417,
+      "loss": 1.6724,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.2818536758422852,
+      "rewards/margins": 0.0854635089635849,
+      "rewards/rejected": -1.3673170804977417,
+      "step": 1545
+    },
+    {
+      "epoch": 0.8295701622344873,
+      "grad_norm": 8.037816116604773,
+      "learning_rate": 9.080666951296276e-07,
+      "logits/chosen": -0.1481930911540985,
+      "logits/rejected": 0.08756627142429352,
+      "logps/chosen": -1.3378673791885376,
+      "logps/rejected": -1.4493402242660522,
+      "loss": 1.7021,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.3378673791885376,
+      "rewards/margins": 0.11147283017635345,
+      "rewards/rejected": -1.4493402242660522,
+      "step": 1550
+    },
+    {
+      "epoch": 0.8322461950158889,
+      "grad_norm": 4.843415681313186,
+      "learning_rate": 9.071647508110305e-07,
+      "logits/chosen": -0.11743097007274628,
+      "logits/rejected": 0.09829775989055634,
+      "logps/chosen": -1.3695207834243774,
+      "logps/rejected": -1.5013995170593262,
+      "loss": 1.7087,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.3695207834243774,
+      "rewards/margins": 0.1318788379430771,
+      "rewards/rejected": -1.5013995170593262,
+      "step": 1555
+    },
+    {
+      "epoch": 0.8349222277972905,
+      "grad_norm": 7.42188862659024,
+      "learning_rate": 9.062588561782354e-07,
+      "logits/chosen": -0.01619536615908146,
+      "logits/rejected": 0.039568670094013214,
+      "logps/chosen": -1.3216458559036255,
+      "logps/rejected": -1.4633032083511353,
+      "loss": 1.6597,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.3216458559036255,
+      "rewards/margins": 0.14165739715099335,
+      "rewards/rejected": -1.4633032083511353,
+      "step": 1560
+    },
+    {
+      "epoch": 0.8375982605786921,
+      "grad_norm": 6.469587220819448,
+      "learning_rate": 9.053490200202358e-07,
+      "logits/chosen": -0.016276555135846138,
+      "logits/rejected": 0.06772230565547943,
+      "logps/chosen": -1.3213629722595215,
+      "logps/rejected": -1.4255964756011963,
+      "loss": 1.687,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.3213629722595215,
+      "rewards/margins": 0.10423363745212555,
+      "rewards/rejected": -1.4255964756011963,
+      "step": 1565
+    },
+    {
+      "epoch": 0.8402742933600936,
+      "grad_norm": 7.895324039537961,
+      "learning_rate": 9.044352511642661e-07,
+      "logits/chosen": 0.025027606636285782,
+      "logits/rejected": 0.029301051050424576,
+      "logps/chosen": -1.219848871231079,
+      "logps/rejected": -1.3602640628814697,
+      "loss": 1.6118,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.219848871231079,
+      "rewards/margins": 0.14041535556316376,
+      "rewards/rejected": -1.3602640628814697,
+      "step": 1570
+    },
+    {
+      "epoch": 0.8429503261414952,
+      "grad_norm": 8.004495331877422,
+      "learning_rate": 9.03517558475716e-07,
+      "logits/chosen": -0.043523408472537994,
+      "logits/rejected": 0.042294926941394806,
+      "logps/chosen": -1.2938941717147827,
+      "logps/rejected": -1.3521257638931274,
+      "loss": 1.6673,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.2938941717147827,
+      "rewards/margins": 0.058231599628925323,
+      "rewards/rejected": -1.3521257638931274,
+      "step": 1575
+    },
+    {
+      "epoch": 0.8456263589228968,
+      "grad_norm": 6.624735016894455,
+      "learning_rate": 9.025959508580436e-07,
+      "logits/chosen": 0.030850157141685486,
+      "logits/rejected": 0.24259766936302185,
+      "logps/chosen": -1.286088228225708,
+      "logps/rejected": -1.421175241470337,
+      "loss": 1.6249,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.286088228225708,
+      "rewards/margins": 0.13508695363998413,
+      "rewards/rejected": -1.421175241470337,
+      "step": 1580
+    },
+    {
+      "epoch": 0.8483023917042983,
+      "grad_norm": 6.966093077574928,
+      "learning_rate": 9.016704372526905e-07,
+      "logits/chosen": -0.01990427076816559,
+      "logits/rejected": 0.11765620857477188,
+      "logps/chosen": -1.2193325757980347,
+      "logps/rejected": -1.467350721359253,
+      "loss": 1.5908,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.2193325757980347,
+      "rewards/margins": 0.24801822006702423,
+      "rewards/rejected": -1.467350721359253,
+      "step": 1585
+    },
+    {
+      "epoch": 0.8509784244856999,
+      "grad_norm": 7.458677237507632,
+      "learning_rate": 9.007410266389934e-07,
+      "logits/chosen": -0.07480472326278687,
+      "logits/rejected": 0.004531972110271454,
+      "logps/chosen": -1.2420049905776978,
+      "logps/rejected": -1.3440052270889282,
+      "loss": 1.6286,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.2420049905776978,
+      "rewards/margins": 0.10200023651123047,
+      "rewards/rejected": -1.3440052270889282,
+      "step": 1590
+    },
+    {
+      "epoch": 0.8536544572671015,
+      "grad_norm": 9.279232253571516,
+      "learning_rate": 8.998077280340981e-07,
+      "logits/chosen": -0.017351821064949036,
+      "logits/rejected": 0.05029697343707085,
+      "logps/chosen": -1.3600581884384155,
+      "logps/rejected": -1.3599737882614136,
+      "loss": 1.7314,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -1.3600581884384155,
+      "rewards/margins": -8.444488048553467e-05,
+      "rewards/rejected": -1.3599737882614136,
+      "step": 1595
+    },
+    {
+      "epoch": 0.8563304900485031,
+      "grad_norm": 6.8897238465074215,
+      "learning_rate": 8.988705504928722e-07,
+      "logits/chosen": -0.13428761065006256,
+      "logits/rejected": 0.0201809611171484,
+      "logps/chosen": -1.3263137340545654,
+      "logps/rejected": -1.500237226486206,
+      "loss": 1.6569,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.3263137340545654,
+      "rewards/margins": 0.17392337322235107,
+      "rewards/rejected": -1.500237226486206,
+      "step": 1600
+    },
+    {
+      "epoch": 0.8563304900485031,
+      "eval_logits/chosen": 0.18784047663211823,
+      "eval_logits/rejected": 0.2617562711238861,
+      "eval_logps/chosen": -1.3093810081481934,
+      "eval_logps/rejected": -1.4585061073303223,
+      "eval_loss": 1.6597652435302734,
+      "eval_rewards/accuracies": 0.5593471527099609,
+      "eval_rewards/chosen": -1.3093810081481934,
+      "eval_rewards/margins": 0.14912521839141846,
+      "eval_rewards/rejected": -1.4585061073303223,
+      "eval_runtime": 40.2695,
+      "eval_samples_per_second": 33.4,
+      "eval_steps_per_second": 8.369,
+      "step": 1600
+    },
+    {
+      "epoch": 0.8590065228299046,
+      "grad_norm": 5.203434488845097,
+      "learning_rate": 8.979295031078157e-07,
+      "logits/chosen": -0.12461533397436142,
+      "logits/rejected": 0.062472231686115265,
+      "logps/chosen": -1.2665091753005981,
+      "logps/rejected": -1.4833637475967407,
+      "loss": 1.6001,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.2665091753005981,
+      "rewards/margins": 0.21685440838336945,
+      "rewards/rejected": -1.4833637475967407,
+      "step": 1605
+    },
+    {
+      "epoch": 0.8616825556113062,
+      "grad_norm": 5.831112519526011,
+      "learning_rate": 8.969845950089751e-07,
+      "logits/chosen": -0.1279621422290802,
+      "logits/rejected": 0.02661943808197975,
+      "logps/chosen": -1.2545493841171265,
+      "logps/rejected": -1.4126601219177246,
+      "loss": 1.6204,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.2545493841171265,
+      "rewards/margins": 0.1581106185913086,
+      "rewards/rejected": -1.4126601219177246,
+      "step": 1610
+    },
+    {
+      "epoch": 0.8643585883927078,
+      "grad_norm": 7.223114448938468,
+      "learning_rate": 8.960358353638526e-07,
+      "logits/chosen": -0.0668589398264885,
+      "logits/rejected": 0.021599432453513145,
+      "logps/chosen": -1.3225297927856445,
+      "logps/rejected": -1.5004407167434692,
+      "loss": 1.6714,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.3225297927856445,
+      "rewards/margins": 0.17791098356246948,
+      "rewards/rejected": -1.5004407167434692,
+      "step": 1615
+    },
+    {
+      "epoch": 0.8670346211741093,
+      "grad_norm": 6.852298379018131,
+      "learning_rate": 8.950832333773184e-07,
+      "logits/chosen": -0.015650823712348938,
+      "logits/rejected": 0.10268108546733856,
+      "logps/chosen": -1.19065260887146,
+      "logps/rejected": -1.4471497535705566,
+      "loss": 1.554,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.19065260887146,
+      "rewards/margins": 0.2564970850944519,
+      "rewards/rejected": -1.4471497535705566,
+      "step": 1620
+    },
+    {
+      "epoch": 0.869710653955511,
+      "grad_norm": 8.250958293931829,
+      "learning_rate": 8.941267982915213e-07,
+      "logits/chosen": 0.06653941422700882,
+      "logits/rejected": 0.11267844587564468,
+      "logps/chosen": -1.3536145687103271,
+      "logps/rejected": -1.4748936891555786,
+      "loss": 1.7003,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.3536145687103271,
+      "rewards/margins": 0.12127907574176788,
+      "rewards/rejected": -1.4748936891555786,
+      "step": 1625
+    },
+    {
+      "epoch": 0.8723866867369126,
+      "grad_norm": 7.516947224333936,
+      "learning_rate": 8.931665393857983e-07,
+      "logits/chosen": 0.02407361939549446,
+      "logits/rejected": 0.16263404488563538,
+      "logps/chosen": -1.311852216720581,
+      "logps/rejected": -1.3916914463043213,
+      "loss": 1.6763,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.311852216720581,
+      "rewards/margins": 0.07983918488025665,
+      "rewards/rejected": -1.3916914463043213,
+      "step": 1630
+    },
+    {
+      "epoch": 0.875062719518314,
+      "grad_norm": 11.051621336233081,
+      "learning_rate": 8.922024659765861e-07,
+      "logits/chosen": -0.08223044127225876,
+      "logits/rejected": 0.019241278991103172,
+      "logps/chosen": -1.2253283262252808,
+      "logps/rejected": -1.378430724143982,
+      "loss": 1.6116,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.2253283262252808,
+      "rewards/margins": 0.15310248732566833,
+      "rewards/rejected": -1.378430724143982,
+      "step": 1635
+    },
+    {
+      "epoch": 0.8777387522997157,
+      "grad_norm": 6.939402757279656,
+      "learning_rate": 8.912345874173288e-07,
+      "logits/chosen": -0.073940709233284,
+      "logits/rejected": 0.01882219687104225,
+      "logps/chosen": -1.2232308387756348,
+      "logps/rejected": -1.3898383378982544,
+      "loss": 1.6271,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.2232308387756348,
+      "rewards/margins": 0.1666075587272644,
+      "rewards/rejected": -1.3898383378982544,
+      "step": 1640
+    },
+    {
+      "epoch": 0.8804147850811173,
+      "grad_norm": 10.12377883475872,
+      "learning_rate": 8.902629130983885e-07,
+      "logits/chosen": -0.04563382267951965,
+      "logits/rejected": -0.0019987802952528,
+      "logps/chosen": -1.2227944135665894,
+      "logps/rejected": -1.366692304611206,
+      "loss": 1.6104,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.2227944135665894,
+      "rewards/margins": 0.1438979208469391,
+      "rewards/rejected": -1.366692304611206,
+      "step": 1645
+    },
+    {
+      "epoch": 0.8830908178625189,
+      "grad_norm": 11.195083737144992,
+      "learning_rate": 8.892874524469537e-07,
+      "logits/chosen": 0.01719006523489952,
+      "logits/rejected": 0.07226898521184921,
+      "logps/chosen": -1.2502089738845825,
+      "logps/rejected": -1.4320770502090454,
+      "loss": 1.6281,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.2502089738845825,
+      "rewards/margins": 0.18186809122562408,
+      "rewards/rejected": -1.4320770502090454,
+      "step": 1650
+    },
+    {
+      "epoch": 0.8857668506439204,
+      "grad_norm": 10.210347433071322,
+      "learning_rate": 8.883082149269478e-07,
+      "logits/chosen": -0.10980989784002304,
+      "logits/rejected": -0.023154709488153458,
+      "logps/chosen": -1.3029086589813232,
+      "logps/rejected": -1.3899312019348145,
+      "loss": 1.7001,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.3029086589813232,
+      "rewards/margins": 0.08702269196510315,
+      "rewards/rejected": -1.3899312019348145,
+      "step": 1655
+    },
+    {
+      "epoch": 0.888442883425322,
+      "grad_norm": 6.521466025549045,
+      "learning_rate": 8.873252100389377e-07,
+      "logits/chosen": -0.021430622786283493,
+      "logits/rejected": -0.02687462605535984,
+      "logps/chosen": -1.2348066568374634,
+      "logps/rejected": -1.4026113748550415,
+      "loss": 1.6231,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.2348066568374634,
+      "rewards/margins": 0.167804554104805,
+      "rewards/rejected": -1.4026113748550415,
+      "step": 1660
+    },
+    {
+      "epoch": 0.8911189162067236,
+      "grad_norm": 6.715722948427614,
+      "learning_rate": 8.863384473200411e-07,
+      "logits/chosen": -0.07429514080286026,
+      "logits/rejected": -0.014555975794792175,
+      "logps/chosen": -1.3120548725128174,
+      "logps/rejected": -1.4360677003860474,
+      "loss": 1.6682,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.3120548725128174,
+      "rewards/margins": 0.1240130066871643,
+      "rewards/rejected": -1.4360677003860474,
+      "step": 1665
+    },
+    {
+      "epoch": 0.8937949489881251,
+      "grad_norm": 6.370698833816654,
+      "learning_rate": 8.853479363438342e-07,
+      "logits/chosen": -0.030076956376433372,
+      "logits/rejected": 0.10825265944004059,
+      "logps/chosen": -1.3282768726348877,
+      "logps/rejected": -1.3937517404556274,
+      "loss": 1.7328,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -1.3282768726348877,
+      "rewards/margins": 0.0654749944806099,
+      "rewards/rejected": -1.3937517404556274,
+      "step": 1670
+    },
+    {
+      "epoch": 0.8964709817695267,
+      "grad_norm": 5.216425362326687,
+      "learning_rate": 8.843536867202588e-07,
+      "logits/chosen": -0.03181172162294388,
+      "logits/rejected": 0.1464720368385315,
+      "logps/chosen": -1.3180739879608154,
+      "logps/rejected": -1.5350233316421509,
+      "loss": 1.6808,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.3180739879608154,
+      "rewards/margins": 0.21694926917552948,
+      "rewards/rejected": -1.5350233316421509,
+      "step": 1675
+    },
+    {
+      "epoch": 0.8991470145509283,
+      "grad_norm": 7.1767344877196395,
+      "learning_rate": 8.833557080955292e-07,
+      "logits/chosen": -0.12470666319131851,
+      "logits/rejected": -0.026866847649216652,
+      "logps/chosen": -1.346660852432251,
+      "logps/rejected": -1.456504225730896,
+      "loss": 1.6877,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.346660852432251,
+      "rewards/margins": 0.10984358936548233,
+      "rewards/rejected": -1.456504225730896,
+      "step": 1680
+    },
+    {
+      "epoch": 0.9018230473323299,
+      "grad_norm": 8.837202279598182,
+      "learning_rate": 8.823540101520381e-07,
+      "logits/chosen": -0.11130370944738388,
+      "logits/rejected": 0.0831967443227768,
+      "logps/chosen": -1.2970563173294067,
+      "logps/rejected": -1.4199358224868774,
+      "loss": 1.6593,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.2970563173294067,
+      "rewards/margins": 0.12287943065166473,
+      "rewards/rejected": -1.4199358224868774,
+      "step": 1685
+    },
+    {
+      "epoch": 0.9044990801137314,
+      "grad_norm": 7.101092718565959,
+      "learning_rate": 8.813486026082637e-07,
+      "logits/chosen": -0.09423160552978516,
+      "logits/rejected": 0.0658324807882309,
+      "logps/chosen": -1.2349854707717896,
+      "logps/rejected": -1.4053113460540771,
+      "loss": 1.5942,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.2349854707717896,
+      "rewards/margins": 0.1703261435031891,
+      "rewards/rejected": -1.4053113460540771,
+      "step": 1690
+    },
+    {
+      "epoch": 0.907175112895133,
+      "grad_norm": 11.380726305489514,
+      "learning_rate": 8.803394952186742e-07,
+      "logits/chosen": -0.22291898727416992,
+      "logits/rejected": -0.10640467703342438,
+      "logps/chosen": -1.3459924459457397,
+      "logps/rejected": -1.4340465068817139,
+      "loss": 1.6934,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.3459924459457397,
+      "rewards/margins": 0.08805397152900696,
+      "rewards/rejected": -1.4340465068817139,
+      "step": 1695
+    },
+    {
+      "epoch": 0.9098511456765346,
+      "grad_norm": 8.462265361008011,
+      "learning_rate": 8.793266977736342e-07,
+      "logits/chosen": -0.07550237327814102,
+      "logits/rejected": -0.108060821890831,
+      "logps/chosen": -1.3348376750946045,
+      "logps/rejected": -1.3659391403198242,
+      "loss": 1.7038,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.3348376750946045,
+      "rewards/margins": 0.031101513653993607,
+      "rewards/rejected": -1.3659391403198242,
+      "step": 1700
+    },
+    {
+      "epoch": 0.9125271784579361,
+      "grad_norm": 8.429614512956558,
+      "learning_rate": 8.783102200993085e-07,
+      "logits/chosen": -0.05549921840429306,
+      "logits/rejected": 0.06219468265771866,
+      "logps/chosen": -1.3043372631072998,
+      "logps/rejected": -1.3859697580337524,
+      "loss": 1.6764,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.3043372631072998,
+      "rewards/margins": 0.08163245767354965,
+      "rewards/rejected": -1.3859697580337524,
+      "step": 1705
+    },
+    {
+      "epoch": 0.9152032112393377,
+      "grad_norm": 7.277943305755828,
+      "learning_rate": 8.772900720575683e-07,
+      "logits/chosen": -0.08484108000993729,
+      "logits/rejected": -0.018476814031600952,
+      "logps/chosen": -1.2428280115127563,
+      "logps/rejected": -1.3736469745635986,
+      "loss": 1.6092,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.2428280115127563,
+      "rewards/margins": 0.13081900775432587,
+      "rewards/rejected": -1.3736469745635986,
+      "step": 1710
+    },
+    {
+      "epoch": 0.9178792440207393,
+      "grad_norm": 6.586317878710247,
+      "learning_rate": 8.762662635458944e-07,
+      "logits/chosen": -0.05393046885728836,
+      "logits/rejected": 0.13396093249320984,
+      "logps/chosen": -1.341209053993225,
+      "logps/rejected": -1.4041177034378052,
+      "loss": 1.7144,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.341209053993225,
+      "rewards/margins": 0.06290876865386963,
+      "rewards/rejected": -1.4041177034378052,
+      "step": 1715
+    },
+    {
+      "epoch": 0.9205552768021408,
+      "grad_norm": 6.562643912935402,
+      "learning_rate": 8.752388044972811e-07,
+      "logits/chosen": -0.05539485067129135,
+      "logits/rejected": -0.0037333101499825716,
+      "logps/chosen": -1.1808788776397705,
+      "logps/rejected": -1.4093412160873413,
+      "loss": 1.5732,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.1808788776397705,
+      "rewards/margins": 0.22846241295337677,
+      "rewards/rejected": -1.4093412160873413,
+      "step": 1720
+    },
+    {
+      "epoch": 0.9232313095835424,
+      "grad_norm": 6.781443857743248,
+      "learning_rate": 8.74207704880141e-07,
+      "logits/chosen": -0.09590543806552887,
+      "logits/rejected": -0.013217410072684288,
+      "logps/chosen": -1.315438985824585,
+      "logps/rejected": -1.5088715553283691,
+      "loss": 1.6445,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.315438985824585,
+      "rewards/margins": 0.19343265891075134,
+      "rewards/rejected": -1.5088715553283691,
+      "step": 1725
+    },
+    {
+      "epoch": 0.925907342364944,
+      "grad_norm": 9.628500492354721,
+      "learning_rate": 8.731729746982068e-07,
+      "logits/chosen": -0.014389969408512115,
+      "logits/rejected": 0.02294905297458172,
+      "logps/chosen": -1.2849242687225342,
+      "logps/rejected": -1.3661396503448486,
+      "loss": 1.6693,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.2849242687225342,
+      "rewards/margins": 0.08121520280838013,
+      "rewards/rejected": -1.3661396503448486,
+      "step": 1730
+    },
+    {
+      "epoch": 0.9285833751463456,
+      "grad_norm": 7.225324232180578,
+      "learning_rate": 8.721346239904355e-07,
+      "logits/chosen": -0.1296091377735138,
+      "logits/rejected": -0.0007158033549785614,
+      "logps/chosen": -1.219684362411499,
+      "logps/rejected": -1.5547126531600952,
+      "loss": 1.5572,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.219684362411499,
+      "rewards/margins": 0.33502835035324097,
+      "rewards/rejected": -1.5547126531600952,
+      "step": 1735
+    },
+    {
+      "epoch": 0.9312594079277471,
+      "grad_norm": 6.684078242296239,
+      "learning_rate": 8.710926628309101e-07,
+      "logits/chosen": -0.10116715729236603,
+      "logits/rejected": 0.00490850955247879,
+      "logps/chosen": -1.2695972919464111,
+      "logps/rejected": -1.3845787048339844,
+      "loss": 1.6351,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.2695972919464111,
+      "rewards/margins": 0.11498130857944489,
+      "rewards/rejected": -1.3845787048339844,
+      "step": 1740
+    },
+    {
+      "epoch": 0.9339354407091487,
+      "grad_norm": 6.928267081500885,
+      "learning_rate": 8.700471013287424e-07,
+      "logits/chosen": -0.04399807006120682,
+      "logits/rejected": -0.02890801429748535,
+      "logps/chosen": -1.2690376043319702,
+      "logps/rejected": -1.3916754722595215,
+      "loss": 1.6372,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.2690376043319702,
+      "rewards/margins": 0.12263789027929306,
+      "rewards/rejected": -1.3916754722595215,
+      "step": 1745
+    },
+    {
+      "epoch": 0.9366114734905503,
+      "grad_norm": 10.873837266637638,
+      "learning_rate": 8.689979496279746e-07,
+      "logits/chosen": -0.09588148444890976,
+      "logits/rejected": -0.04271275922656059,
+      "logps/chosen": -1.267203450202942,
+      "logps/rejected": -1.4820235967636108,
+      "loss": 1.6022,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.267203450202942,
+      "rewards/margins": 0.21482017636299133,
+      "rewards/rejected": -1.4820235967636108,
+      "step": 1750
+    },
+    {
+      "epoch": 0.9392875062719518,
+      "grad_norm": 5.799402693567249,
+      "learning_rate": 8.679452179074811e-07,
+      "logits/chosen": -0.10049313306808472,
+      "logits/rejected": -0.03091563656926155,
+      "logps/chosen": -1.2542965412139893,
+      "logps/rejected": -1.3590319156646729,
+      "loss": 1.6401,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.2542965412139893,
+      "rewards/margins": 0.1047353744506836,
+      "rewards/rejected": -1.3590319156646729,
+      "step": 1755
+    },
+    {
+      "epoch": 0.9419635390533534,
+      "grad_norm": 6.570245639505618,
+      "learning_rate": 8.668889163808698e-07,
+      "logits/chosen": -0.04846472293138504,
+      "logits/rejected": 0.04894987493753433,
+      "logps/chosen": -1.2414864301681519,
+      "logps/rejected": -1.3920495510101318,
+      "loss": 1.6123,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.2414864301681519,
+      "rewards/margins": 0.15056300163269043,
+      "rewards/rejected": -1.3920495510101318,
+      "step": 1760
+    },
+    {
+      "epoch": 0.944639571834755,
+      "grad_norm": 6.5262857057583945,
+      "learning_rate": 8.658290552963827e-07,
+      "logits/chosen": -0.07380203902721405,
+      "logits/rejected": -0.06247757002711296,
+      "logps/chosen": -1.2712262868881226,
+      "logps/rejected": -1.428794503211975,
+      "loss": 1.6511,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.2712262868881226,
+      "rewards/margins": 0.15756818652153015,
+      "rewards/rejected": -1.428794503211975,
+      "step": 1765
+    },
+    {
+      "epoch": 0.9473156046161565,
+      "grad_norm": 5.698089935456351,
+      "learning_rate": 8.647656449367966e-07,
+      "logits/chosen": -0.07224474102258682,
+      "logits/rejected": 0.056889694184064865,
+      "logps/chosen": -1.2968194484710693,
+      "logps/rejected": -1.3564350605010986,
+      "loss": 1.6635,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.2968194484710693,
+      "rewards/margins": 0.059615641832351685,
+      "rewards/rejected": -1.3564350605010986,
+      "step": 1770
+    },
+    {
+      "epoch": 0.9499916373975581,
+      "grad_norm": 8.571788954671964,
+      "learning_rate": 8.636986956193235e-07,
+      "logits/chosen": -0.10607154667377472,
+      "logits/rejected": -0.044310472905635834,
+      "logps/chosen": -1.2398325204849243,
+      "logps/rejected": -1.3684263229370117,
+      "loss": 1.6144,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.2398325204849243,
+      "rewards/margins": 0.12859384715557098,
+      "rewards/rejected": -1.3684263229370117,
+      "step": 1775
+    },
+    {
+      "epoch": 0.9526676701789597,
+      "grad_norm": 6.323104023838116,
+      "learning_rate": 8.626282176955104e-07,
+      "logits/chosen": -0.10209281742572784,
+      "logits/rejected": -0.0055378819815814495,
+      "logps/chosen": -1.2686989307403564,
+      "logps/rejected": -1.4309422969818115,
+      "loss": 1.6258,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.2686989307403564,
+      "rewards/margins": 0.16224347054958344,
+      "rewards/rejected": -1.4309422969818115,
+      "step": 1780
+    },
+    {
+      "epoch": 0.9553437029603613,
+      "grad_norm": 7.238973153918211,
+      "learning_rate": 8.615542215511389e-07,
+      "logits/chosen": -0.0030011646449565887,
+      "logits/rejected": 0.05824938416481018,
+      "logps/chosen": -1.2141025066375732,
+      "logps/rejected": -1.3099324703216553,
+      "loss": 1.6379,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.2141025066375732,
+      "rewards/margins": 0.09582982957363129,
+      "rewards/rejected": -1.3099324703216553,
+      "step": 1785
+    },
+    {
+      "epoch": 0.9580197357417628,
+      "grad_norm": 7.513938150549295,
+      "learning_rate": 8.604767176061241e-07,
+      "logits/chosen": -0.01581764593720436,
+      "logits/rejected": 0.01219199039041996,
+      "logps/chosen": -1.2912495136260986,
+      "logps/rejected": -1.4009100198745728,
+      "loss": 1.6528,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.2912495136260986,
+      "rewards/margins": 0.10966048389673233,
+      "rewards/rejected": -1.4009100198745728,
+      "step": 1790
+    },
+    {
+      "epoch": 0.9606957685231644,
+      "grad_norm": 7.930001609958226,
+      "learning_rate": 8.593957163144141e-07,
+      "logits/chosen": -0.10950282961130142,
+      "logits/rejected": 0.006719866301864386,
+      "logps/chosen": -1.2600187063217163,
+      "logps/rejected": -1.407183289527893,
+      "loss": 1.628,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.2600187063217163,
+      "rewards/margins": 0.1471645087003708,
+      "rewards/rejected": -1.407183289527893,
+      "step": 1795
+    },
+    {
+      "epoch": 0.963371801304566,
+      "grad_norm": 5.2855450798311345,
+      "learning_rate": 8.58311228163888e-07,
+      "logits/chosen": -0.048804882913827896,
+      "logits/rejected": 0.008072657510638237,
+      "logps/chosen": -1.2580320835113525,
+      "logps/rejected": -1.348109245300293,
+      "loss": 1.6488,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.2580320835113525,
+      "rewards/margins": 0.09007705003023148,
+      "rewards/rejected": -1.348109245300293,
+      "step": 1800
+    },
+    {
+      "epoch": 0.9660478340859675,
+      "grad_norm": 5.961444422853368,
+      "learning_rate": 8.57223263676255e-07,
+      "logits/chosen": -0.17057205736637115,
+      "logits/rejected": -0.06418205797672272,
+      "logps/chosen": -1.2190537452697754,
+      "logps/rejected": -1.4326839447021484,
+      "loss": 1.5672,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.2190537452697754,
+      "rewards/margins": 0.21363027393817902,
+      "rewards/rejected": -1.4326839447021484,
+      "step": 1805
+    },
+    {
+      "epoch": 0.9687238668673691,
+      "grad_norm": 7.644020895661337,
+      "learning_rate": 8.561318334069511e-07,
+      "logits/chosen": -0.07568531483411789,
+      "logits/rejected": 0.04486411437392235,
+      "logps/chosen": -1.2447948455810547,
+      "logps/rejected": -1.3525478839874268,
+      "loss": 1.6348,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.2447948455810547,
+      "rewards/margins": 0.10775311291217804,
+      "rewards/rejected": -1.3525478839874268,
+      "step": 1810
+    },
+    {
+      "epoch": 0.9713998996487707,
+      "grad_norm": 6.165154965788141,
+      "learning_rate": 8.550369479450375e-07,
+      "logits/chosen": -0.08772541582584381,
+      "logits/rejected": 0.012555623427033424,
+      "logps/chosen": -1.2673677206039429,
+      "logps/rejected": -1.3656182289123535,
+      "loss": 1.6303,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.2673677206039429,
+      "rewards/margins": 0.09825052320957184,
+      "rewards/rejected": -1.3656182289123535,
+      "step": 1815
+    },
+    {
+      "epoch": 0.9740759324301723,
+      "grad_norm": 7.377835800463116,
+      "learning_rate": 8.539386179130977e-07,
+      "logits/chosen": -0.0693771094083786,
+      "logits/rejected": -0.01106178667396307,
+      "logps/chosen": -1.25356125831604,
+      "logps/rejected": -1.3627688884735107,
+      "loss": 1.6323,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.25356125831604,
+      "rewards/margins": 0.10920742899179459,
+      "rewards/rejected": -1.3627688884735107,
+      "step": 1820
+    },
+    {
+      "epoch": 0.9767519652115738,
+      "grad_norm": 7.934772543113003,
+      "learning_rate": 8.528368539671347e-07,
+      "logits/chosen": -0.11911240965127945,
+      "logits/rejected": -0.001425326569005847,
+      "logps/chosen": -1.2460615634918213,
+      "logps/rejected": -1.41835618019104,
+      "loss": 1.6204,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.2460615634918213,
+      "rewards/margins": 0.1722945123910904,
+      "rewards/rejected": -1.41835618019104,
+      "step": 1825
+    },
+    {
+      "epoch": 0.9794279979929754,
+      "grad_norm": 6.007705828704846,
+      "learning_rate": 8.51731666796467e-07,
+      "logits/chosen": 0.026783233508467674,
+      "logits/rejected": 0.04644298553466797,
+      "logps/chosen": -1.3235788345336914,
+      "logps/rejected": -1.358957052230835,
+      "loss": 1.7024,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.3235788345336914,
+      "rewards/margins": 0.03537829965353012,
+      "rewards/rejected": -1.358957052230835,
+      "step": 1830
+    },
+    {
+      "epoch": 0.982104030774377,
+      "grad_norm": 7.568868917842844,
+      "learning_rate": 8.506230671236254e-07,
+      "logits/chosen": -0.10422410070896149,
+      "logits/rejected": -0.06238463521003723,
+      "logps/chosen": -1.2823209762573242,
+      "logps/rejected": -1.3079733848571777,
+      "loss": 1.6726,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.2823209762573242,
+      "rewards/margins": 0.02565241977572441,
+      "rewards/rejected": -1.3079733848571777,
+      "step": 1835
+    },
+    {
+      "epoch": 0.9847800635557785,
+      "grad_norm": 6.762765506914262,
+      "learning_rate": 8.495110657042488e-07,
+      "logits/chosen": -0.013921190984547138,
+      "logits/rejected": 0.07133986055850983,
+      "logps/chosen": -1.278881311416626,
+      "logps/rejected": -1.4670178890228271,
+      "loss": 1.6263,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.278881311416626,
+      "rewards/margins": 0.18813663721084595,
+      "rewards/rejected": -1.4670178890228271,
+      "step": 1840
+    },
+    {
+      "epoch": 0.9874560963371801,
+      "grad_norm": 9.237192200041854,
+      "learning_rate": 8.483956733269799e-07,
+      "logits/chosen": -0.09804896265268326,
+      "logits/rejected": -0.021493665874004364,
+      "logps/chosen": -1.3015546798706055,
+      "logps/rejected": -1.3693385124206543,
+      "loss": 1.6763,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -1.3015546798706055,
+      "rewards/margins": 0.06778378784656525,
+      "rewards/rejected": -1.3693385124206543,
+      "step": 1845
+    },
+    {
+      "epoch": 0.9901321291185817,
+      "grad_norm": 6.283239648580291,
+      "learning_rate": 8.472769008133602e-07,
+      "logits/chosen": -0.19651444256305695,
+      "logits/rejected": -0.09052984416484833,
+      "logps/chosen": -1.3121778964996338,
+      "logps/rejected": -1.3153350353240967,
+      "loss": 1.692,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -1.3121778964996338,
+      "rewards/margins": 0.003157214028760791,
+      "rewards/rejected": -1.3153350353240967,
+      "step": 1850
+    },
+    {
+      "epoch": 0.9928081618999832,
+      "grad_norm": 6.419885578451689,
+      "learning_rate": 8.461547590177259e-07,
+      "logits/chosen": -0.0700322762131691,
+      "logits/rejected": 0.012923069298267365,
+      "logps/chosen": -1.2328972816467285,
+      "logps/rejected": -1.3694627285003662,
+      "loss": 1.6283,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.2328972816467285,
+      "rewards/margins": 0.13656553626060486,
+      "rewards/rejected": -1.3694627285003662,
+      "step": 1855
+    },
+    {
+      "epoch": 0.9954841946813848,
+      "grad_norm": 7.90071898604423,
+      "learning_rate": 8.450292588271014e-07,
+      "logits/chosen": -0.06717083603143692,
+      "logits/rejected": -0.01019772794097662,
+      "logps/chosen": -1.3151053190231323,
+      "logps/rejected": -1.4005153179168701,
+      "loss": 1.6626,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.3151053190231323,
+      "rewards/margins": 0.08540989458560944,
+      "rewards/rejected": -1.4005153179168701,
+      "step": 1860
+    },
+    {
+      "epoch": 0.9981602274627864,
+      "grad_norm": 6.849933687630706,
+      "learning_rate": 8.439004111610945e-07,
+      "logits/chosen": -0.07957644760608673,
+      "logits/rejected": -0.012797405943274498,
+      "logps/chosen": -1.1850707530975342,
+      "logps/rejected": -1.4084447622299194,
+      "loss": 1.5734,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.1850707530975342,
+      "rewards/margins": 0.22337403893470764,
+      "rewards/rejected": -1.4084447622299194,
+      "step": 1865
+    },
+    {
+      "epoch": 1.000836260244188,
+      "grad_norm": 7.127300215535271,
+      "learning_rate": 8.427682269717901e-07,
+      "logits/chosen": -0.12439294904470444,
+      "logits/rejected": 0.008108256384730339,
+      "logps/chosen": -1.329679012298584,
+      "logps/rejected": -1.378037452697754,
+      "loss": 1.6945,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -1.329679012298584,
+      "rewards/margins": 0.04835844039916992,
+      "rewards/rejected": -1.378037452697754,
+      "step": 1870
+    },
+    {
+      "epoch": 1.0035122930255895,
+      "grad_norm": 5.767377885020671,
+      "learning_rate": 8.416327172436446e-07,
+      "logits/chosen": -0.1611921340227127,
+      "logits/rejected": -0.04769011586904526,
+      "logps/chosen": -1.3013145923614502,
+      "logps/rejected": -1.3885616064071655,
+      "loss": 1.6582,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.3013145923614502,
+      "rewards/margins": 0.0872470885515213,
+      "rewards/rejected": -1.3885616064071655,
+      "step": 1875
+    },
+    {
+      "epoch": 1.0061883258069912,
+      "grad_norm": 7.775122590944508,
+      "learning_rate": 8.404938929933778e-07,
+      "logits/chosen": -0.04842232167720795,
+      "logits/rejected": 0.10100684314966202,
+      "logps/chosen": -1.2551026344299316,
+      "logps/rejected": -1.5187945365905762,
+      "loss": 1.6163,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.2551026344299316,
+      "rewards/margins": 0.2636919319629669,
+      "rewards/rejected": -1.5187945365905762,
+      "step": 1880
+    },
+    {
+      "epoch": 1.0088643585883927,
+      "grad_norm": 7.56258171453072,
+      "learning_rate": 8.39351765269868e-07,
+      "logits/chosen": -0.08237516134977341,
+      "logits/rejected": -0.016337528824806213,
+      "logps/chosen": -1.212425947189331,
+      "logps/rejected": -1.3873404264450073,
+      "loss": 1.5975,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.212425947189331,
+      "rewards/margins": 0.17491456866264343,
+      "rewards/rejected": -1.3873404264450073,
+      "step": 1885
+    },
+    {
+      "epoch": 1.0115403913697942,
+      "grad_norm": 6.432102596180142,
+      "learning_rate": 8.382063451540431e-07,
+      "logits/chosen": -0.08515187352895737,
+      "logits/rejected": 0.08776731789112091,
+      "logps/chosen": -1.2676560878753662,
+      "logps/rejected": -1.41463303565979,
+      "loss": 1.6642,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.2676560878753662,
+      "rewards/margins": 0.146977037191391,
+      "rewards/rejected": -1.41463303565979,
+      "step": 1890
+    },
+    {
+      "epoch": 1.014216424151196,
+      "grad_norm": 7.346269675416436,
+      "learning_rate": 8.370576437587742e-07,
+      "logits/chosen": -0.03652515262365341,
+      "logits/rejected": 0.013286001980304718,
+      "logps/chosen": -1.2494899034500122,
+      "logps/rejected": -1.3886620998382568,
+      "loss": 1.6114,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.2494899034500122,
+      "rewards/margins": 0.13917216658592224,
+      "rewards/rejected": -1.3886620998382568,
+      "step": 1895
+    },
+    {
+      "epoch": 1.0168924569325974,
+      "grad_norm": 7.157047781782498,
+      "learning_rate": 8.359056722287674e-07,
+      "logits/chosen": -0.144033744931221,
+      "logits/rejected": 0.07644566893577576,
+      "logps/chosen": -1.2893970012664795,
+      "logps/rejected": -1.3897775411605835,
+      "loss": 1.6615,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.2893970012664795,
+      "rewards/margins": 0.10038057714700699,
+      "rewards/rejected": -1.3897775411605835,
+      "step": 1900
+    },
+    {
+      "epoch": 1.019568489713999,
+      "grad_norm": 6.635519431216746,
+      "learning_rate": 8.347504417404553e-07,
+      "logits/chosen": -0.07440408319234848,
+      "logits/rejected": 0.04772048071026802,
+      "logps/chosen": -1.3085484504699707,
+      "logps/rejected": -1.4066262245178223,
+      "loss": 1.6716,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.3085484504699707,
+      "rewards/margins": 0.09807775914669037,
+      "rewards/rejected": -1.4066262245178223,
+      "step": 1905
+    },
+    {
+      "epoch": 1.0222445224954007,
+      "grad_norm": 7.46897956636028,
+      "learning_rate": 8.335919635018893e-07,
+      "logits/chosen": -0.1354013830423355,
+      "logits/rejected": -0.029260188341140747,
+      "logps/chosen": -1.272361397743225,
+      "logps/rejected": -1.4245942831039429,
+      "loss": 1.646,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.272361397743225,
+      "rewards/margins": 0.1522330790758133,
+      "rewards/rejected": -1.4245942831039429,
+      "step": 1910
+    },
+    {
+      "epoch": 1.0249205552768021,
+      "grad_norm": 5.021632127702659,
+      "learning_rate": 8.324302487526303e-07,
+      "logits/chosen": -0.0943860188126564,
+      "logits/rejected": -0.033584389835596085,
+      "logps/chosen": -1.237389326095581,
+      "logps/rejected": -1.335540533065796,
+      "loss": 1.6227,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.237389326095581,
+      "rewards/margins": 0.09815128147602081,
+      "rewards/rejected": -1.335540533065796,
+      "step": 1915
+    },
+    {
+      "epoch": 1.0275965880582036,
+      "grad_norm": 6.477177263733326,
+      "learning_rate": 8.312653087636398e-07,
+      "logits/chosen": -0.11647417396306992,
+      "logits/rejected": -0.0515231117606163,
+      "logps/chosen": -1.1594212055206299,
+      "logps/rejected": -1.3588650226593018,
+      "loss": 1.5432,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.1594212055206299,
+      "rewards/margins": 0.19944405555725098,
+      "rewards/rejected": -1.3588650226593018,
+      "step": 1920
+    },
+    {
+      "epoch": 1.0302726208396054,
+      "grad_norm": 7.798364232564433,
+      "learning_rate": 8.300971548371711e-07,
+      "logits/chosen": -0.23913152515888214,
+      "logits/rejected": -0.065396249294281,
+      "logps/chosen": -1.3392633199691772,
+      "logps/rejected": -1.415614128112793,
+      "loss": 1.6877,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.3392633199691772,
+      "rewards/margins": 0.07635075598955154,
+      "rewards/rejected": -1.415614128112793,
+      "step": 1925
+    },
+    {
+      "epoch": 1.0329486536210069,
+      "grad_norm": 6.679655849773325,
+      "learning_rate": 8.289257983066582e-07,
+      "logits/chosen": -0.16242703795433044,
+      "logits/rejected": -0.0466943196952343,
+      "logps/chosen": -1.2052810192108154,
+      "logps/rejected": -1.3960282802581787,
+      "loss": 1.5734,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.2052810192108154,
+      "rewards/margins": 0.19074717164039612,
+      "rewards/rejected": -1.3960282802581787,
+      "step": 1930
+    },
+    {
+      "epoch": 1.0356246864024083,
+      "grad_norm": 6.569896272550323,
+      "learning_rate": 8.277512505366077e-07,
+      "logits/chosen": -0.17269453406333923,
+      "logits/rejected": -0.024066952988505363,
+      "logps/chosen": -1.2725989818572998,
+      "logps/rejected": -1.4294071197509766,
+      "loss": 1.6212,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.2725989818572998,
+      "rewards/margins": 0.15680822730064392,
+      "rewards/rejected": -1.4294071197509766,
+      "step": 1935
+    },
+    {
+      "epoch": 1.03830071918381,
+      "grad_norm": 7.3011255358224805,
+      "learning_rate": 8.265735229224868e-07,
+      "logits/chosen": -0.07453256845474243,
+      "logits/rejected": 0.019622694700956345,
+      "logps/chosen": -1.277066946029663,
+      "logps/rejected": -1.4361835718154907,
+      "loss": 1.6421,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.277066946029663,
+      "rewards/margins": 0.15911665558815002,
+      "rewards/rejected": -1.4361835718154907,
+      "step": 1940
+    },
+    {
+      "epoch": 1.0409767519652116,
+      "grad_norm": 7.008864997055822,
+      "learning_rate": 8.253926268906144e-07,
+      "logits/chosen": -0.18078279495239258,
+      "logits/rejected": -0.037943147122859955,
+      "logps/chosen": -1.2641736268997192,
+      "logps/rejected": -1.4041342735290527,
+      "loss": 1.6017,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.2641736268997192,
+      "rewards/margins": 0.1399606615304947,
+      "rewards/rejected": -1.4041342735290527,
+      "step": 1945
+    },
+    {
+      "epoch": 1.043652784746613,
+      "grad_norm": 6.158357754291056,
+      "learning_rate": 8.242085738980487e-07,
+      "logits/chosen": -0.10943479835987091,
+      "logits/rejected": 0.0673285573720932,
+      "logps/chosen": -1.3101437091827393,
+      "logps/rejected": -1.4342368841171265,
+      "loss": 1.6474,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.3101437091827393,
+      "rewards/margins": 0.12409321218729019,
+      "rewards/rejected": -1.4342368841171265,
+      "step": 1950
+    },
+    {
+      "epoch": 1.0463288175280148,
+      "grad_norm": 7.083286067561728,
+      "learning_rate": 8.230213754324772e-07,
+      "logits/chosen": -0.05732116848230362,
+      "logits/rejected": 0.0017935142386704683,
+      "logps/chosen": -1.2026760578155518,
+      "logps/rejected": -1.4104280471801758,
+      "loss": 1.5549,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.2026760578155518,
+      "rewards/margins": 0.20775175094604492,
+      "rewards/rejected": -1.4104280471801758,
+      "step": 1955
+    },
+    {
+      "epoch": 1.0490048503094163,
+      "grad_norm": 6.817524204988108,
+      "learning_rate": 8.218310430121045e-07,
+      "logits/chosen": -0.14348937571048737,
+      "logits/rejected": -0.12298564612865448,
+      "logps/chosen": -1.26016104221344,
+      "logps/rejected": -1.3855254650115967,
+      "loss": 1.6371,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.26016104221344,
+      "rewards/margins": 0.12536440789699554,
+      "rewards/rejected": -1.3855254650115967,
+      "step": 1960
+    },
+    {
+      "epoch": 1.051680883090818,
+      "grad_norm": 7.530792490915361,
+      "learning_rate": 8.20637588185541e-07,
+      "logits/chosen": -0.04563091695308685,
+      "logits/rejected": 0.011556325480341911,
+      "logps/chosen": -1.2070486545562744,
+      "logps/rejected": -1.4804352521896362,
+      "loss": 1.5382,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.2070486545562744,
+      "rewards/margins": 0.2733866274356842,
+      "rewards/rejected": -1.4804352521896362,
+      "step": 1965
+    },
+    {
+      "epoch": 1.0543569158722195,
+      "grad_norm": 8.37658162909406,
+      "learning_rate": 8.194410225316906e-07,
+      "logits/chosen": -0.10722582042217255,
+      "logits/rejected": 0.012183678336441517,
+      "logps/chosen": -1.271067500114441,
+      "logps/rejected": -1.4413100481033325,
+      "loss": 1.6331,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.271067500114441,
+      "rewards/margins": 0.1702425330877304,
+      "rewards/rejected": -1.4413100481033325,
+      "step": 1970
+    },
+    {
+      "epoch": 1.057032948653621,
+      "grad_norm": 7.464246482195032,
+      "learning_rate": 8.182413576596385e-07,
+      "logits/chosen": -0.059704482555389404,
+      "logits/rejected": 0.018035439774394035,
+      "logps/chosen": -1.2023093700408936,
+      "logps/rejected": -1.3934221267700195,
+      "loss": 1.567,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.2023093700408936,
+      "rewards/margins": 0.19111260771751404,
+      "rewards/rejected": -1.3934221267700195,
+      "step": 1975
+    },
+    {
+      "epoch": 1.0597089814350227,
+      "grad_norm": 7.578940543456657,
+      "learning_rate": 8.170386052085389e-07,
+      "logits/chosen": -0.016566650941967964,
+      "logits/rejected": 0.09114401787519455,
+      "logps/chosen": -1.275981068611145,
+      "logps/rejected": -1.4322946071624756,
+      "loss": 1.6423,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.275981068611145,
+      "rewards/margins": 0.15631352365016937,
+      "rewards/rejected": -1.4322946071624756,
+      "step": 1980
+    },
+    {
+      "epoch": 1.0623850142164242,
+      "grad_norm": 7.700307637421773,
+      "learning_rate": 8.158327768475008e-07,
+      "logits/chosen": -0.08740787953138351,
+      "logits/rejected": 0.04793049767613411,
+      "logps/chosen": -1.298185110092163,
+      "logps/rejected": -1.3765360116958618,
+      "loss": 1.6752,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.298185110092163,
+      "rewards/margins": 0.07835081964731216,
+      "rewards/rejected": -1.3765360116958618,
+      "step": 1985
+    },
+    {
+      "epoch": 1.0650610469978257,
+      "grad_norm": 7.481636127449876,
+      "learning_rate": 8.146238842754767e-07,
+      "logits/chosen": -0.11278965324163437,
+      "logits/rejected": -0.03838568925857544,
+      "logps/chosen": -1.3026186227798462,
+      "logps/rejected": -1.4424206018447876,
+      "loss": 1.6447,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.3026186227798462,
+      "rewards/margins": 0.13980190455913544,
+      "rewards/rejected": -1.4424206018447876,
+      "step": 1990
+    },
+    {
+      "epoch": 1.0677370797792274,
+      "grad_norm": 7.134876372667975,
+      "learning_rate": 8.134119392211476e-07,
+      "logits/chosen": -0.012486773543059826,
+      "logits/rejected": 0.11474494636058807,
+      "logps/chosen": -1.2178418636322021,
+      "logps/rejected": -1.470642328262329,
+      "loss": 1.5763,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.2178418636322021,
+      "rewards/margins": 0.2528002858161926,
+      "rewards/rejected": -1.470642328262329,
+      "step": 1995
+    },
+    {
+      "epoch": 1.0704131125606289,
+      "grad_norm": 11.370675419727878,
+      "learning_rate": 8.121969534428094e-07,
+      "logits/chosen": -0.11129634082317352,
+      "logits/rejected": 0.017330098897218704,
+      "logps/chosen": -1.301973581314087,
+      "logps/rejected": -1.3558018207550049,
+      "loss": 1.7111,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -1.301973581314087,
+      "rewards/margins": 0.05382819101214409,
+      "rewards/rejected": -1.3558018207550049,
+      "step": 2000
+    },
+    {
+      "epoch": 1.0704131125606289,
+      "eval_logits/chosen": 0.15607893466949463,
+      "eval_logits/rejected": 0.22898563742637634,
+      "eval_logps/chosen": -1.3001796007156372,
+      "eval_logps/rejected": -1.4570097923278809,
+      "eval_loss": 1.6547960042953491,
+      "eval_rewards/accuracies": 0.5652819275856018,
+      "eval_rewards/chosen": -1.3001796007156372,
+      "eval_rewards/margins": 0.1568300873041153,
+      "eval_rewards/rejected": -1.4570097923278809,
+      "eval_runtime": 40.3494,
+      "eval_samples_per_second": 33.334,
+      "eval_steps_per_second": 8.352,
+      "step": 2000
+    },
+    {
+      "epoch": 1.0730891453420304,
+      "grad_norm": 8.28139468174092,
+      "learning_rate": 8.109789387282599e-07,
+      "logits/chosen": -0.07982680201530457,
+      "logits/rejected": -0.008802761323750019,
+      "logps/chosen": -1.3038156032562256,
+      "logps/rejected": -1.3784399032592773,
+      "loss": 1.6776,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.3038156032562256,
+      "rewards/margins": 0.07462429255247116,
+      "rewards/rejected": -1.3784399032592773,
+      "step": 2005
+    },
+    {
+      "epoch": 1.075765178123432,
+      "grad_norm": 9.331971218931889,
+      "learning_rate": 8.097579068946827e-07,
+      "logits/chosen": -0.04008934646844864,
+      "logits/rejected": 0.04494168609380722,
+      "logps/chosen": -1.2365713119506836,
+      "logps/rejected": -1.3746750354766846,
+      "loss": 1.6245,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.2365713119506836,
+      "rewards/margins": 0.13810376822948456,
+      "rewards/rejected": -1.3746750354766846,
+      "step": 2010
+    },
+    {
+      "epoch": 1.0784412109048336,
+      "grad_norm": 6.722924785559778,
+      "learning_rate": 8.085338697885344e-07,
+      "logits/chosen": -0.09236650913953781,
+      "logits/rejected": 0.04134202376008034,
+      "logps/chosen": -1.2058128118515015,
+      "logps/rejected": -1.3785531520843506,
+      "loss": 1.5911,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.2058128118515015,
+      "rewards/margins": 0.17274019122123718,
+      "rewards/rejected": -1.3785531520843506,
+      "step": 2015
+    },
+    {
+      "epoch": 1.081117243686235,
+      "grad_norm": 7.269682600004079,
+      "learning_rate": 8.073068392854282e-07,
+      "logits/chosen": -0.11385257542133331,
+      "logits/rejected": 0.02374332770705223,
+      "logps/chosen": -1.3075402975082397,
+      "logps/rejected": -1.4661028385162354,
+      "loss": 1.6516,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.3075402975082397,
+      "rewards/margins": 0.15856240689754486,
+      "rewards/rejected": -1.4661028385162354,
+      "step": 2020
+    },
+    {
+      "epoch": 1.0837932764676368,
+      "grad_norm": 7.28315693106649,
+      "learning_rate": 8.060768272900193e-07,
+      "logits/chosen": -0.034406136721372604,
+      "logits/rejected": 0.0682801604270935,
+      "logps/chosen": -1.2798516750335693,
+      "logps/rejected": -1.4447687864303589,
+      "loss": 1.6364,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.2798516750335693,
+      "rewards/margins": 0.16491718590259552,
+      "rewards/rejected": -1.4447687864303589,
+      "step": 2025
+    },
+    {
+      "epoch": 1.0864693092490383,
+      "grad_norm": 6.330620780904875,
+      "learning_rate": 8.0484384573589e-07,
+      "logits/chosen": -0.11100749671459198,
+      "logits/rejected": -0.0917942225933075,
+      "logps/chosen": -1.224623680114746,
+      "logps/rejected": -1.3809386491775513,
+      "loss": 1.6082,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.224623680114746,
+      "rewards/margins": 0.15631495416164398,
+      "rewards/rejected": -1.3809386491775513,
+      "step": 2030
+    },
+    {
+      "epoch": 1.0891453420304398,
+      "grad_norm": 6.963132320526369,
+      "learning_rate": 8.03607906585432e-07,
+      "logits/chosen": -0.1358167678117752,
+      "logits/rejected": 0.003392726182937622,
+      "logps/chosen": -1.2268271446228027,
+      "logps/rejected": -1.369664192199707,
+      "loss": 1.5988,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.2268271446228027,
+      "rewards/margins": 0.14283707737922668,
+      "rewards/rejected": -1.369664192199707,
+      "step": 2035
+    },
+    {
+      "epoch": 1.0918213748118415,
+      "grad_norm": 8.895581081854896,
+      "learning_rate": 8.023690218297329e-07,
+      "logits/chosen": -0.1667802482843399,
+      "logits/rejected": -0.11317966133356094,
+      "logps/chosen": -1.244923710823059,
+      "logps/rejected": -1.3557780981063843,
+      "loss": 1.6213,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.244923710823059,
+      "rewards/margins": 0.11085452139377594,
+      "rewards/rejected": -1.3557780981063843,
+      "step": 2040
+    },
+    {
+      "epoch": 1.094497407593243,
+      "grad_norm": 10.612181651266052,
+      "learning_rate": 8.01127203488458e-07,
+      "logits/chosen": -0.033369529992341995,
+      "logits/rejected": 0.00568109517917037,
+      "logps/chosen": -1.2427600622177124,
+      "logps/rejected": -1.4127317667007446,
+      "loss": 1.6157,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.2427600622177124,
+      "rewards/margins": 0.16997160017490387,
+      "rewards/rejected": -1.4127317667007446,
+      "step": 2045
+    },
+    {
+      "epoch": 1.0971734403746445,
+      "grad_norm": 8.355761392932418,
+      "learning_rate": 7.998824636097339e-07,
+      "logits/chosen": -0.1496758908033371,
+      "logits/rejected": -0.055368490517139435,
+      "logps/chosen": -1.2817127704620361,
+      "logps/rejected": -1.3591550588607788,
+      "loss": 1.6769,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.2817127704620361,
+      "rewards/margins": 0.07744228094816208,
+      "rewards/rejected": -1.3591550588607788,
+      "step": 2050
+    },
+    {
+      "epoch": 1.0998494731560462,
+      "grad_norm": 8.360987746496702,
+      "learning_rate": 7.986348142700328e-07,
+      "logits/chosen": -0.06102384254336357,
+      "logits/rejected": 0.03928098827600479,
+      "logps/chosen": -1.25723397731781,
+      "logps/rejected": -1.3665558099746704,
+      "loss": 1.6262,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.25723397731781,
+      "rewards/margins": 0.10932165384292603,
+      "rewards/rejected": -1.3665558099746704,
+      "step": 2055
+    },
+    {
+      "epoch": 1.1025255059374477,
+      "grad_norm": 8.116749314927926,
+      "learning_rate": 7.973842675740539e-07,
+      "logits/chosen": -0.06331825256347656,
+      "logits/rejected": -0.025836337357759476,
+      "logps/chosen": -1.283013939857483,
+      "logps/rejected": -1.4757953882217407,
+      "loss": 1.6254,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.283013939857483,
+      "rewards/margins": 0.1927814930677414,
+      "rewards/rejected": -1.4757953882217407,
+      "step": 2060
+    },
+    {
+      "epoch": 1.1052015387188494,
+      "grad_norm": 7.775977833971184,
+      "learning_rate": 7.961308356546066e-07,
+      "logits/chosen": -0.09919768571853638,
+      "logits/rejected": 0.0063897669315338135,
+      "logps/chosen": -1.2619423866271973,
+      "logps/rejected": -1.3673630952835083,
+      "loss": 1.6427,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.2619423866271973,
+      "rewards/margins": 0.10542067140340805,
+      "rewards/rejected": -1.3673630952835083,
+      "step": 2065
+    },
+    {
+      "epoch": 1.107877571500251,
+      "grad_norm": 8.689507448407964,
+      "learning_rate": 7.948745306724931e-07,
+      "logits/chosen": -0.12621857225894928,
+      "logits/rejected": -0.0004501536604948342,
+      "logps/chosen": -1.2156966924667358,
+      "logps/rejected": -1.4456615447998047,
+      "loss": 1.5808,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.2156966924667358,
+      "rewards/margins": 0.22996488213539124,
+      "rewards/rejected": -1.4456615447998047,
+      "step": 2070
+    },
+    {
+      "epoch": 1.1105536042816524,
+      "grad_norm": 7.854038469429008,
+      "learning_rate": 7.936153648163897e-07,
+      "logits/chosen": -0.13651198148727417,
+      "logits/rejected": -0.053521834313869476,
+      "logps/chosen": -1.2578051090240479,
+      "logps/rejected": -1.453272819519043,
+      "loss": 1.6072,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.2578051090240479,
+      "rewards/margins": 0.19546756148338318,
+      "rewards/rejected": -1.453272819519043,
+      "step": 2075
+    },
+    {
+      "epoch": 1.1132296370630541,
+      "grad_norm": 5.696613158096192,
+      "learning_rate": 7.92353350302729e-07,
+      "logits/chosen": -0.18901076912879944,
+      "logits/rejected": -0.05768461897969246,
+      "logps/chosen": -1.1884305477142334,
+      "logps/rejected": -1.4610515832901,
+      "loss": 1.5212,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.1884305477142334,
+      "rewards/margins": 0.2726210355758667,
+      "rewards/rejected": -1.4610515832901,
+      "step": 2080
+    },
+    {
+      "epoch": 1.1159056698444556,
+      "grad_norm": 11.533281987271195,
+      "learning_rate": 7.910884993755816e-07,
+      "logits/chosen": -0.14586128294467926,
+      "logits/rejected": -0.04839394986629486,
+      "logps/chosen": -1.2378690242767334,
+      "logps/rejected": -1.3928966522216797,
+      "loss": 1.6117,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.2378690242767334,
+      "rewards/margins": 0.15502753853797913,
+      "rewards/rejected": -1.3928966522216797,
+      "step": 2085
+    },
+    {
+      "epoch": 1.118581702625857,
+      "grad_norm": 8.700654931151222,
+      "learning_rate": 7.898208243065367e-07,
+      "logits/chosen": -0.21394655108451843,
+      "logits/rejected": -0.2073047161102295,
+      "logps/chosen": -1.2208964824676514,
+      "logps/rejected": -1.3645246028900146,
+      "loss": 1.6088,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.2208964824676514,
+      "rewards/margins": 0.14362801611423492,
+      "rewards/rejected": -1.3645246028900146,
+      "step": 2090
+    },
+    {
+      "epoch": 1.1212577354072588,
+      "grad_norm": 4.874928851192373,
+      "learning_rate": 7.88550337394583e-07,
+      "logits/chosen": -0.12973353266716003,
+      "logits/rejected": -0.014638014137744904,
+      "logps/chosen": -1.368638038635254,
+      "logps/rejected": -1.481255292892456,
+      "loss": 1.7235,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.368638038635254,
+      "rewards/margins": 0.11261717975139618,
+      "rewards/rejected": -1.481255292892456,
+      "step": 2095
+    },
+    {
+      "epoch": 1.1239337681886603,
+      "grad_norm": 8.98215076654875,
+      "learning_rate": 7.872770509659905e-07,
+      "logits/chosen": -0.07033531367778778,
+      "logits/rejected": -0.03589242324233055,
+      "logps/chosen": -1.3756530284881592,
+      "logps/rejected": -1.4444859027862549,
+      "loss": 1.7215,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.3756530284881592,
+      "rewards/margins": 0.06883300095796585,
+      "rewards/rejected": -1.4444859027862549,
+      "step": 2100
+    },
+    {
+      "epoch": 1.1266098009700618,
+      "grad_norm": 7.2182694130911145,
+      "learning_rate": 7.860009773741896e-07,
+      "logits/chosen": -0.04326310753822327,
+      "logits/rejected": 0.051185887306928635,
+      "logps/chosen": -1.3004047870635986,
+      "logps/rejected": -1.425777792930603,
+      "loss": 1.6538,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.3004047870635986,
+      "rewards/margins": 0.12537285685539246,
+      "rewards/rejected": -1.425777792930603,
+      "step": 2105
+    },
+    {
+      "epoch": 1.1292858337514635,
+      "grad_norm": 8.322581432863783,
+      "learning_rate": 7.84722128999652e-07,
+      "logits/chosen": -0.1491938829421997,
+      "logits/rejected": -0.009843626990914345,
+      "logps/chosen": -1.2436976432800293,
+      "logps/rejected": -1.5600513219833374,
+      "loss": 1.5737,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.2436976432800293,
+      "rewards/margins": 0.31635379791259766,
+      "rewards/rejected": -1.5600513219833374,
+      "step": 2110
+    },
+    {
+      "epoch": 1.131961866532865,
+      "grad_norm": 7.902528377969128,
+      "learning_rate": 7.834405182497699e-07,
+      "logits/chosen": -0.03493956848978996,
+      "logits/rejected": 0.010392325930297375,
+      "logps/chosen": -1.248529314994812,
+      "logps/rejected": -1.376386046409607,
+      "loss": 1.6132,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.248529314994812,
+      "rewards/margins": 0.12785673141479492,
+      "rewards/rejected": -1.376386046409607,
+      "step": 2115
+    },
+    {
+      "epoch": 1.1346378993142665,
+      "grad_norm": 8.606169225182978,
+      "learning_rate": 7.821561575587368e-07,
+      "logits/chosen": -0.15363888442516327,
+      "logits/rejected": -0.12825623154640198,
+      "logps/chosen": -1.2925937175750732,
+      "logps/rejected": -1.417511224746704,
+      "loss": 1.6548,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.2925937175750732,
+      "rewards/margins": 0.12491756677627563,
+      "rewards/rejected": -1.417511224746704,
+      "step": 2120
+    },
+    {
+      "epoch": 1.1373139320956682,
+      "grad_norm": 6.966458401018398,
+      "learning_rate": 7.808690593874254e-07,
+      "logits/chosen": -0.10021853446960449,
+      "logits/rejected": -0.054100703448057175,
+      "logps/chosen": -1.2008590698242188,
+      "logps/rejected": -1.4212961196899414,
+      "loss": 1.5527,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.2008590698242188,
+      "rewards/margins": 0.2204369753599167,
+      "rewards/rejected": -1.4212961196899414,
+      "step": 2125
+    },
+    {
+      "epoch": 1.1399899648770697,
+      "grad_norm": 8.922400788653567,
+      "learning_rate": 7.79579236223268e-07,
+      "logits/chosen": -0.067740797996521,
+      "logits/rejected": 0.1286747008562088,
+      "logps/chosen": -1.2899023294448853,
+      "logps/rejected": -1.4537795782089233,
+      "loss": 1.6364,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.2899023294448853,
+      "rewards/margins": 0.1638774424791336,
+      "rewards/rejected": -1.4537795782089233,
+      "step": 2130
+    },
+    {
+      "epoch": 1.1426659976584714,
+      "grad_norm": 6.447264543939956,
+      "learning_rate": 7.782867005801346e-07,
+      "logits/chosen": -0.07721833139657974,
+      "logits/rejected": 0.0497472882270813,
+      "logps/chosen": -1.2961310148239136,
+      "logps/rejected": -1.446725845336914,
+      "loss": 1.6488,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.2961310148239136,
+      "rewards/margins": 0.1505947858095169,
+      "rewards/rejected": -1.446725845336914,
+      "step": 2135
+    },
+    {
+      "epoch": 1.145342030439873,
+      "grad_norm": 7.918047521721078,
+      "learning_rate": 7.769914649982117e-07,
+      "logits/chosen": -0.1346278041601181,
+      "logits/rejected": -0.022326147183775902,
+      "logps/chosen": -1.2685073614120483,
+      "logps/rejected": -1.3912893533706665,
+      "loss": 1.6397,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.2685073614120483,
+      "rewards/margins": 0.12278201431035995,
+      "rewards/rejected": -1.3912893533706665,
+      "step": 2140
+    },
+    {
+      "epoch": 1.1480180632212744,
+      "grad_norm": 7.802641619948372,
+      "learning_rate": 7.756935420438803e-07,
+      "logits/chosen": -0.10416577756404877,
+      "logits/rejected": -0.04111819341778755,
+      "logps/chosen": -1.1541697978973389,
+      "logps/rejected": -1.391998052597046,
+      "loss": 1.5353,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.1541697978973389,
+      "rewards/margins": 0.2378282994031906,
+      "rewards/rejected": -1.391998052597046,
+      "step": 2145
+    },
+    {
+      "epoch": 1.1506940960026761,
+      "grad_norm": 5.455496619518153,
+      "learning_rate": 7.743929443095951e-07,
+      "logits/chosen": -0.15382114052772522,
+      "logits/rejected": -0.1144246906042099,
+      "logps/chosen": -1.3197849988937378,
+      "logps/rejected": -1.4354331493377686,
+      "loss": 1.6704,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.3197849988937378,
+      "rewards/margins": 0.11564798653125763,
+      "rewards/rejected": -1.4354331493377686,
+      "step": 2150
+    },
+    {
+      "epoch": 1.1533701287840776,
+      "grad_norm": 6.590797479707265,
+      "learning_rate": 7.730896844137609e-07,
+      "logits/chosen": -0.07442931830883026,
+      "logits/rejected": -0.023158835247159004,
+      "logps/chosen": -1.30672287940979,
+      "logps/rejected": -1.4486448764801025,
+      "loss": 1.6368,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.30672287940979,
+      "rewards/margins": 0.14192210137844086,
+      "rewards/rejected": -1.4486448764801025,
+      "step": 2155
+    },
+    {
+      "epoch": 1.1560461615654791,
+      "grad_norm": 8.71553571729924,
+      "learning_rate": 7.717837750006106e-07,
+      "logits/chosen": -0.16045820713043213,
+      "logits/rejected": -0.06852944940328598,
+      "logps/chosen": -1.2256841659545898,
+      "logps/rejected": -1.4260807037353516,
+      "loss": 1.593,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.2256841659545898,
+      "rewards/margins": 0.20039649307727814,
+      "rewards/rejected": -1.4260807037353516,
+      "step": 2160
+    },
+    {
+      "epoch": 1.1587221943468808,
+      "grad_norm": 7.010858616243839,
+      "learning_rate": 7.704752287400832e-07,
+      "logits/chosen": -0.13092279434204102,
+      "logits/rejected": 0.020101290196180344,
+      "logps/chosen": -1.2686822414398193,
+      "logps/rejected": -1.4782660007476807,
+      "loss": 1.6078,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.2686822414398193,
+      "rewards/margins": 0.2095838040113449,
+      "rewards/rejected": -1.4782660007476807,
+      "step": 2165
+    },
+    {
+      "epoch": 1.1613982271282823,
+      "grad_norm": 5.457383139654073,
+      "learning_rate": 7.691640583277004e-07,
+      "logits/chosen": -0.1236150711774826,
+      "logits/rejected": 0.0086661446839571,
+      "logps/chosen": -1.2275776863098145,
+      "logps/rejected": -1.46425199508667,
+      "loss": 1.5892,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.2275776863098145,
+      "rewards/margins": 0.23667418956756592,
+      "rewards/rejected": -1.46425199508667,
+      "step": 2170
+    },
+    {
+      "epoch": 1.1640742599096838,
+      "grad_norm": 6.136029451743161,
+      "learning_rate": 7.678502764844433e-07,
+      "logits/chosen": -0.12522700428962708,
+      "logits/rejected": 0.016509711742401123,
+      "logps/chosen": -1.3003489971160889,
+      "logps/rejected": -1.386040210723877,
+      "loss": 1.6649,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.3003489971160889,
+      "rewards/margins": 0.08569102734327316,
+      "rewards/rejected": -1.386040210723877,
+      "step": 2175
+    },
+    {
+      "epoch": 1.1667502926910855,
+      "grad_norm": 6.081976073123297,
+      "learning_rate": 7.665338959566288e-07,
+      "logits/chosen": -0.12515996396541595,
+      "logits/rejected": -0.06724338978528976,
+      "logps/chosen": -1.2342045307159424,
+      "logps/rejected": -1.4075241088867188,
+      "loss": 1.597,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.2342045307159424,
+      "rewards/margins": 0.17331981658935547,
+      "rewards/rejected": -1.4075241088867188,
+      "step": 2180
+    },
+    {
+      "epoch": 1.169426325472487,
+      "grad_norm": 8.431364114046154,
+      "learning_rate": 7.652149295157868e-07,
+      "logits/chosen": -0.05369647592306137,
+      "logits/rejected": 0.05816897749900818,
+      "logps/chosen": -1.265222430229187,
+      "logps/rejected": -1.3789777755737305,
+      "loss": 1.62,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.265222430229187,
+      "rewards/margins": 0.11375533044338226,
+      "rewards/rejected": -1.3789777755737305,
+      "step": 2185
+    },
+    {
+      "epoch": 1.1721023582538885,
+      "grad_norm": 6.915142843947729,
+      "learning_rate": 7.638933899585354e-07,
+      "logits/chosen": 0.02304958924651146,
+      "logits/rejected": 0.06372608244419098,
+      "logps/chosen": -1.2534650564193726,
+      "logps/rejected": -1.3916088342666626,
+      "loss": 1.6261,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.2534650564193726,
+      "rewards/margins": 0.13814368844032288,
+      "rewards/rejected": -1.3916088342666626,
+      "step": 2190
+    },
+    {
+      "epoch": 1.1747783910352902,
+      "grad_norm": 8.9950441140299,
+      "learning_rate": 7.625692901064573e-07,
+      "logits/chosen": -0.040085647255182266,
+      "logits/rejected": 0.0382084995508194,
+      "logps/chosen": -1.2309796810150146,
+      "logps/rejected": -1.4908417463302612,
+      "loss": 1.5839,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.2309796810150146,
+      "rewards/margins": 0.2598620057106018,
+      "rewards/rejected": -1.4908417463302612,
+      "step": 2195
+    },
+    {
+      "epoch": 1.1774544238166917,
+      "grad_norm": 7.528852900763409,
+      "learning_rate": 7.61242642805975e-07,
+      "logits/chosen": -0.14972008764743805,
+      "logits/rejected": -0.15985627472400665,
+      "logps/chosen": -1.2626259326934814,
+      "logps/rejected": -1.4132511615753174,
+      "loss": 1.6235,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.2626259326934814,
+      "rewards/margins": 0.15062522888183594,
+      "rewards/rejected": -1.4132511615753174,
+      "step": 2200
+    },
+    {
+      "epoch": 1.1801304565980932,
+      "grad_norm": 7.013133418167818,
+      "learning_rate": 7.599134609282266e-07,
+      "logits/chosen": -0.14835958182811737,
+      "logits/rejected": 0.0036022409331053495,
+      "logps/chosen": -1.199286699295044,
+      "logps/rejected": -1.3756859302520752,
+      "loss": 1.5803,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.199286699295044,
+      "rewards/margins": 0.17639943957328796,
+      "rewards/rejected": -1.3756859302520752,
+      "step": 2205
+    },
+    {
+      "epoch": 1.182806489379495,
+      "grad_norm": 8.446358358562367,
+      "learning_rate": 7.585817573689402e-07,
+      "logits/chosen": -0.2204298973083496,
+      "logits/rejected": -0.1284269541501999,
+      "logps/chosen": -1.1430498361587524,
+      "logps/rejected": -1.395293951034546,
+      "loss": 1.5234,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.1430498361587524,
+      "rewards/margins": 0.2522440552711487,
+      "rewards/rejected": -1.395293951034546,
+      "step": 2210
+    },
+    {
+      "epoch": 1.1854825221608964,
+      "grad_norm": 7.8713142227663635,
+      "learning_rate": 7.572475450483098e-07,
+      "logits/chosen": -0.16724872589111328,
+      "logits/rejected": -0.10567456483840942,
+      "logps/chosen": -1.3644531965255737,
+      "logps/rejected": -1.530957818031311,
+      "loss": 1.6859,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.3644531965255737,
+      "rewards/margins": 0.1665046066045761,
+      "rewards/rejected": -1.530957818031311,
+      "step": 2215
+    },
+    {
+      "epoch": 1.188158554942298,
+      "grad_norm": 6.236928008931055,
+      "learning_rate": 7.559108369108689e-07,
+      "logits/chosen": -0.22434313595294952,
+      "logits/rejected": -0.12110327184200287,
+      "logps/chosen": -1.2065577507019043,
+      "logps/rejected": -1.3525969982147217,
+      "loss": 1.5948,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.2065577507019043,
+      "rewards/margins": 0.14603909850120544,
+      "rewards/rejected": -1.3525969982147217,
+      "step": 2220
+    },
+    {
+      "epoch": 1.1908345877236997,
+      "grad_norm": 8.20692761506631,
+      "learning_rate": 7.54571645925366e-07,
+      "logits/chosen": -0.18120639026165009,
+      "logits/rejected": -0.02124299295246601,
+      "logps/chosen": -1.2259933948516846,
+      "logps/rejected": -1.461756706237793,
+      "loss": 1.5631,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.2259933948516846,
+      "rewards/margins": 0.2357633411884308,
+      "rewards/rejected": -1.461756706237793,
+      "step": 2225
+    },
+    {
+      "epoch": 1.1935106205051011,
+      "grad_norm": 13.378601723429425,
+      "learning_rate": 7.532299850846378e-07,
+      "logits/chosen": -0.23142382502555847,
+      "logits/rejected": -0.1267291009426117,
+      "logps/chosen": -1.2854366302490234,
+      "logps/rejected": -1.5767123699188232,
+      "loss": 1.634,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.2854366302490234,
+      "rewards/margins": 0.29127568006515503,
+      "rewards/rejected": -1.5767123699188232,
+      "step": 2230
+    },
+    {
+      "epoch": 1.1961866532865026,
+      "grad_norm": 9.090806417248851,
+      "learning_rate": 7.518858674054838e-07,
+      "logits/chosen": -0.17949996888637543,
+      "logits/rejected": -0.02718859352171421,
+      "logps/chosen": -1.2322248220443726,
+      "logps/rejected": -1.4807246923446655,
+      "loss": 1.5928,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.2322248220443726,
+      "rewards/margins": 0.24849990010261536,
+      "rewards/rejected": -1.4807246923446655,
+      "step": 2235
+    },
+    {
+      "epoch": 1.1988626860679044,
+      "grad_norm": 8.873698696072829,
+      "learning_rate": 7.505393059285394e-07,
+      "logits/chosen": -0.18178102374076843,
+      "logits/rejected": -0.06476934254169464,
+      "logps/chosen": -1.2484729290008545,
+      "logps/rejected": -1.45114266872406,
+      "loss": 1.5865,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.2484729290008545,
+      "rewards/margins": 0.2026699334383011,
+      "rewards/rejected": -1.45114266872406,
+      "step": 2240
+    },
+    {
+      "epoch": 1.2015387188493059,
+      "grad_norm": 9.381368374566106,
+      "learning_rate": 7.491903137181501e-07,
+      "logits/chosen": -0.14490970969200134,
+      "logits/rejected": -0.10534831136465073,
+      "logps/chosen": -1.2206541299819946,
+      "logps/rejected": -1.4163111448287964,
+      "loss": 1.6053,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.2206541299819946,
+      "rewards/margins": 0.19565710425376892,
+      "rewards/rejected": -1.4163111448287964,
+      "step": 2245
+    },
+    {
+      "epoch": 1.2042147516307076,
+      "grad_norm": 7.520181553218833,
+      "learning_rate": 7.478389038622441e-07,
+      "logits/chosen": -0.10299573838710785,
+      "logits/rejected": -0.0947377160191536,
+      "logps/chosen": -1.1875255107879639,
+      "logps/rejected": -1.4461685419082642,
+      "loss": 1.5358,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.1875255107879639,
+      "rewards/margins": 0.2586430609226227,
+      "rewards/rejected": -1.4461685419082642,
+      "step": 2250
+    },
+    {
+      "epoch": 1.206890784412109,
+      "grad_norm": 7.006260078296691,
+      "learning_rate": 7.46485089472206e-07,
+      "logits/chosen": -0.15663310885429382,
+      "logits/rejected": -0.06570927798748016,
+      "logps/chosen": -1.3104556798934937,
+      "logps/rejected": -1.3670212030410767,
+      "loss": 1.6854,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.3104556798934937,
+      "rewards/margins": 0.05656537413597107,
+      "rewards/rejected": -1.3670212030410767,
+      "step": 2255
+    },
+    {
+      "epoch": 1.2095668171935106,
+      "grad_norm": 7.18732291044755,
+      "learning_rate": 7.451288836827487e-07,
+      "logits/chosen": -0.09750930964946747,
+      "logits/rejected": -0.11318697780370712,
+      "logps/chosen": -1.2687238454818726,
+      "logps/rejected": -1.3918168544769287,
+      "loss": 1.6193,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.2687238454818726,
+      "rewards/margins": 0.12309300899505615,
+      "rewards/rejected": -1.3918168544769287,
+      "step": 2260
+    },
+    {
+      "epoch": 1.2122428499749123,
+      "grad_norm": 8.435818849093998,
+      "learning_rate": 7.437702996517869e-07,
+      "logits/chosen": -0.1856766641139984,
+      "logits/rejected": -0.10571583360433578,
+      "logps/chosen": -1.3002653121948242,
+      "logps/rejected": -1.4226324558258057,
+      "loss": 1.6478,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.3002653121948242,
+      "rewards/margins": 0.12236703932285309,
+      "rewards/rejected": -1.4226324558258057,
+      "step": 2265
+    },
+    {
+      "epoch": 1.2149188827563138,
+      "grad_norm": 8.438577399522528,
+      "learning_rate": 7.424093505603087e-07,
+      "logits/chosen": -0.275261253118515,
+      "logits/rejected": -0.14463934302330017,
+      "logps/chosen": -1.239253044128418,
+      "logps/rejected": -1.4394651651382446,
+      "loss": 1.5969,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.239253044128418,
+      "rewards/margins": 0.20021216571331024,
+      "rewards/rejected": -1.4394651651382446,
+      "step": 2270
+    },
+    {
+      "epoch": 1.2175949155377153,
+      "grad_norm": 8.260528732786215,
+      "learning_rate": 7.410460496122482e-07,
+      "logits/chosen": -0.1685386598110199,
+      "logits/rejected": -0.0866832584142685,
+      "logps/chosen": -1.2267197370529175,
+      "logps/rejected": -1.4667956829071045,
+      "loss": 1.5713,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.2267197370529175,
+      "rewards/margins": 0.24007587134838104,
+      "rewards/rejected": -1.4667956829071045,
+      "step": 2275
+    },
+    {
+      "epoch": 1.220270948319117,
+      "grad_norm": 10.35057514954781,
+      "learning_rate": 7.396804100343572e-07,
+      "logits/chosen": -0.21187300980091095,
+      "logits/rejected": -0.09693863242864609,
+      "logps/chosen": -1.1686277389526367,
+      "logps/rejected": -1.3477180004119873,
+      "loss": 1.566,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.1686277389526367,
+      "rewards/margins": 0.17909027636051178,
+      "rewards/rejected": -1.3477180004119873,
+      "step": 2280
+    },
+    {
+      "epoch": 1.2229469811005185,
+      "grad_norm": 7.877628525181929,
+      "learning_rate": 7.383124450760768e-07,
+      "logits/chosen": -0.15671268105506897,
+      "logits/rejected": 0.0005567491170950234,
+      "logps/chosen": -1.2920244932174683,
+      "logps/rejected": -1.444446086883545,
+      "loss": 1.6564,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.2920244932174683,
+      "rewards/margins": 0.1524215042591095,
+      "rewards/rejected": -1.444446086883545,
+      "step": 2285
+    },
+    {
+      "epoch": 1.22562301388192,
+      "grad_norm": 9.659680567006552,
+      "learning_rate": 7.369421680094091e-07,
+      "logits/chosen": -0.2353818118572235,
+      "logits/rejected": -0.11391669511795044,
+      "logps/chosen": -1.156830906867981,
+      "logps/rejected": -1.3382948637008667,
+      "loss": 1.563,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.156830906867981,
+      "rewards/margins": 0.18146398663520813,
+      "rewards/rejected": -1.3382948637008667,
+      "step": 2290
+    },
+    {
+      "epoch": 1.2282990466633217,
+      "grad_norm": 7.933801521830124,
+      "learning_rate": 7.355695921287881e-07,
+      "logits/chosen": -0.18124429881572723,
+      "logits/rejected": -0.10757305473089218,
+      "logps/chosen": -1.2433009147644043,
+      "logps/rejected": -1.4033946990966797,
+      "loss": 1.6102,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.2433009147644043,
+      "rewards/margins": 0.16009381413459778,
+      "rewards/rejected": -1.4033946990966797,
+      "step": 2295
+    },
+    {
+      "epoch": 1.2309750794447232,
+      "grad_norm": 8.86717143935084,
+      "learning_rate": 7.341947307509513e-07,
+      "logits/chosen": -0.15329954028129578,
+      "logits/rejected": -0.05239754170179367,
+      "logps/chosen": -1.260248064994812,
+      "logps/rejected": -1.3629111051559448,
+      "loss": 1.6576,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -1.260248064994812,
+      "rewards/margins": 0.10266293585300446,
+      "rewards/rejected": -1.3629111051559448,
+      "step": 2300
+    },
+    {
+      "epoch": 1.233651112226125,
+      "grad_norm": 7.549256858237088,
+      "learning_rate": 7.328175972148094e-07,
+      "logits/chosen": -0.2044523060321808,
+      "logits/rejected": -0.09235817193984985,
+      "logps/chosen": -1.3756862878799438,
+      "logps/rejected": -1.4864490032196045,
+      "loss": 1.717,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.3756862878799438,
+      "rewards/margins": 0.11076265573501587,
+      "rewards/rejected": -1.4864490032196045,
+      "step": 2305
+    },
+    {
+      "epoch": 1.2363271450075264,
+      "grad_norm": 11.536576447700092,
+      "learning_rate": 7.314382048813185e-07,
+      "logits/chosen": -0.1483876258134842,
+      "logits/rejected": 0.06599108129739761,
+      "logps/chosen": -1.2939670085906982,
+      "logps/rejected": -1.4761382341384888,
+      "loss": 1.6266,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.2939670085906982,
+      "rewards/margins": 0.18217109143733978,
+      "rewards/rejected": -1.4761382341384888,
+      "step": 2310
+    },
+    {
+      "epoch": 1.2390031777889279,
+      "grad_norm": 9.100206977273533,
+      "learning_rate": 7.300565671333486e-07,
+      "logits/chosen": -0.14136271178722382,
+      "logits/rejected": -0.005953514017164707,
+      "logps/chosen": -1.271116852760315,
+      "logps/rejected": -1.4551541805267334,
+      "loss": 1.6098,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.271116852760315,
+      "rewards/margins": 0.1840374618768692,
+      "rewards/rejected": -1.4551541805267334,
+      "step": 2315
+    },
+    {
+      "epoch": 1.2416792105703296,
+      "grad_norm": 7.434798313930728,
+      "learning_rate": 7.286726973755554e-07,
+      "logits/chosen": -0.06562301516532898,
+      "logits/rejected": -0.05574440956115723,
+      "logps/chosen": -1.278440237045288,
+      "logps/rejected": -1.4780454635620117,
+      "loss": 1.6292,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.278440237045288,
+      "rewards/margins": 0.19960518181324005,
+      "rewards/rejected": -1.4780454635620117,
+      "step": 2320
+    },
+    {
+      "epoch": 1.244355243351731,
+      "grad_norm": 10.144557930338843,
+      "learning_rate": 7.272866090342493e-07,
+      "logits/chosen": -0.022041847929358482,
+      "logits/rejected": 0.035709965974092484,
+      "logps/chosen": -1.3029844760894775,
+      "logps/rejected": -1.5329054594039917,
+      "loss": 1.642,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.3029844760894775,
+      "rewards/margins": 0.22992126643657684,
+      "rewards/rejected": -1.5329054594039917,
+      "step": 2325
+    },
+    {
+      "epoch": 1.2470312761331326,
+      "grad_norm": 7.631646145563696,
+      "learning_rate": 7.258983155572656e-07,
+      "logits/chosen": -0.20035552978515625,
+      "logits/rejected": -0.11055605113506317,
+      "logps/chosen": -1.258976697921753,
+      "logps/rejected": -1.4475109577178955,
+      "loss": 1.6193,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.258976697921753,
+      "rewards/margins": 0.18853434920310974,
+      "rewards/rejected": -1.4475109577178955,
+      "step": 2330
+    },
+    {
+      "epoch": 1.2497073089145343,
+      "grad_norm": 8.96296132267378,
+      "learning_rate": 7.245078304138335e-07,
+      "logits/chosen": -0.07240384817123413,
+      "logits/rejected": -0.03183193877339363,
+      "logps/chosen": -1.2547897100448608,
+      "logps/rejected": -1.4930450916290283,
+      "loss": 1.5914,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.2547897100448608,
+      "rewards/margins": 0.23825562000274658,
+      "rewards/rejected": -1.4930450916290283,
+      "step": 2335
+    },
+    {
+      "epoch": 1.2523833416959358,
+      "grad_norm": 6.011723470561662,
+      "learning_rate": 7.231151670944462e-07,
+      "logits/chosen": -0.22535963356494904,
+      "logits/rejected": -0.07960195094347,
+      "logps/chosen": -1.270557165145874,
+      "logps/rejected": -1.4250514507293701,
+      "loss": 1.6538,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.270557165145874,
+      "rewards/margins": 0.15449431538581848,
+      "rewards/rejected": -1.4250514507293701,
+      "step": 2340
+    },
+    {
+      "epoch": 1.2550593744773373,
+      "grad_norm": 7.806406519370909,
+      "learning_rate": 7.217203391107291e-07,
+      "logits/chosen": -0.15927395224571228,
+      "logits/rejected": -0.03005305863916874,
+      "logps/chosen": -1.2619166374206543,
+      "logps/rejected": -1.440375566482544,
+      "loss": 1.6392,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.2619166374206543,
+      "rewards/margins": 0.17845895886421204,
+      "rewards/rejected": -1.440375566482544,
+      "step": 2345
+    },
+    {
+      "epoch": 1.257735407258739,
+      "grad_norm": 6.854262667921416,
+      "learning_rate": 7.203233599953096e-07,
+      "logits/chosen": -0.16050459444522858,
+      "logits/rejected": -0.05893082544207573,
+      "logps/chosen": -1.2983603477478027,
+      "logps/rejected": -1.4060739278793335,
+      "loss": 1.6549,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.2983603477478027,
+      "rewards/margins": 0.10771362483501434,
+      "rewards/rejected": -1.4060739278793335,
+      "step": 2350
+    },
+    {
+      "epoch": 1.2604114400401405,
+      "grad_norm": 8.601731390879266,
+      "learning_rate": 7.189242433016852e-07,
+      "logits/chosen": -0.10667651891708374,
+      "logits/rejected": 0.005719064734876156,
+      "logps/chosen": -1.1957563161849976,
+      "logps/rejected": -1.4306471347808838,
+      "loss": 1.5816,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.1957563161849976,
+      "rewards/margins": 0.23489081859588623,
+      "rewards/rejected": -1.4306471347808838,
+      "step": 2355
+    },
+    {
+      "epoch": 1.263087472821542,
+      "grad_norm": 7.030941860681588,
+      "learning_rate": 7.17523002604092e-07,
+      "logits/chosen": -0.0977768674492836,
+      "logits/rejected": -0.0004381418111734092,
+      "logps/chosen": -1.2323318719863892,
+      "logps/rejected": -1.5626273155212402,
+      "loss": 1.577,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.2323318719863892,
+      "rewards/margins": 0.3302956223487854,
+      "rewards/rejected": -1.5626273155212402,
+      "step": 2360
+    },
+    {
+      "epoch": 1.2657635056029437,
+      "grad_norm": 5.815185432245015,
+      "learning_rate": 7.161196514973734e-07,
+      "logits/chosen": -0.08281068503856659,
+      "logits/rejected": 0.02186240255832672,
+      "logps/chosen": -1.2621674537658691,
+      "logps/rejected": -1.4554924964904785,
+      "loss": 1.5928,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.2621674537658691,
+      "rewards/margins": 0.19332513213157654,
+      "rewards/rejected": -1.4554924964904785,
+      "step": 2365
+    },
+    {
+      "epoch": 1.2684395383843452,
+      "grad_norm": 8.139447127312923,
+      "learning_rate": 7.147142035968483e-07,
+      "logits/chosen": -0.07579265534877777,
+      "logits/rejected": 0.01429951936006546,
+      "logps/chosen": -1.2227771282196045,
+      "logps/rejected": -1.4302802085876465,
+      "loss": 1.5711,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.2227771282196045,
+      "rewards/margins": 0.20750324428081512,
+      "rewards/rejected": -1.4302802085876465,
+      "step": 2370
+    },
+    {
+      "epoch": 1.2711155711657467,
+      "grad_norm": 8.398987175821437,
+      "learning_rate": 7.133066725381781e-07,
+      "logits/chosen": -0.21982577443122864,
+      "logits/rejected": -0.0813729465007782,
+      "logps/chosen": -1.175386667251587,
+      "logps/rejected": -1.2754274606704712,
+      "loss": 1.5729,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.175386667251587,
+      "rewards/margins": 0.10004077106714249,
+      "rewards/rejected": -1.2754274606704712,
+      "step": 2375
+    },
+    {
+      "epoch": 1.2737916039471484,
+      "grad_norm": 8.845243873719252,
+      "learning_rate": 7.118970719772354e-07,
+      "logits/chosen": -0.21248643100261688,
+      "logits/rejected": -0.029846271499991417,
+      "logps/chosen": -1.2926652431488037,
+      "logps/rejected": -1.4724124670028687,
+      "loss": 1.6462,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.2926652431488037,
+      "rewards/margins": 0.17974717915058136,
+      "rewards/rejected": -1.4724124670028687,
+      "step": 2380
+    },
+    {
+      "epoch": 1.27646763672855,
+      "grad_norm": 8.305037106630097,
+      "learning_rate": 7.104854155899711e-07,
+      "logits/chosen": -0.0735519751906395,
+      "logits/rejected": 0.012401674874126911,
+      "logps/chosen": -1.2600033283233643,
+      "logps/rejected": -1.42123544216156,
+      "loss": 1.639,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.2600033283233643,
+      "rewards/margins": 0.16123203933238983,
+      "rewards/rejected": -1.42123544216156,
+      "step": 2385
+    },
+    {
+      "epoch": 1.2791436695099514,
+      "grad_norm": 6.713897767599404,
+      "learning_rate": 7.090717170722817e-07,
+      "logits/chosen": -0.10170115530490875,
+      "logits/rejected": -0.053350962698459625,
+      "logps/chosen": -1.2383091449737549,
+      "logps/rejected": -1.4422080516815186,
+      "loss": 1.5887,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.2383091449737549,
+      "rewards/margins": 0.20389895141124725,
+      "rewards/rejected": -1.4422080516815186,
+      "step": 2390
+    },
+    {
+      "epoch": 1.2818197022913531,
+      "grad_norm": 10.19938062205549,
+      "learning_rate": 7.076559901398762e-07,
+      "logits/chosen": -0.259472519159317,
+      "logits/rejected": -0.16848035156726837,
+      "logps/chosen": -1.1729857921600342,
+      "logps/rejected": -1.378514051437378,
+      "loss": 1.5701,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.1729857921600342,
+      "rewards/margins": 0.20552822947502136,
+      "rewards/rejected": -1.378514051437378,
+      "step": 2395
+    },
+    {
+      "epoch": 1.2844957350727546,
+      "grad_norm": 9.317184801020808,
+      "learning_rate": 7.062382485281436e-07,
+      "logits/chosen": -0.09926240146160126,
+      "logits/rejected": -0.007072200533002615,
+      "logps/chosen": -1.2322889566421509,
+      "logps/rejected": -1.4107961654663086,
+      "loss": 1.6123,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.2322889566421509,
+      "rewards/margins": 0.17850707471370697,
+      "rewards/rejected": -1.4107961654663086,
+      "step": 2400
+    },
+    {
+      "epoch": 1.2844957350727546,
+      "eval_logits/chosen": 0.1950339674949646,
+      "eval_logits/rejected": 0.27288389205932617,
+      "eval_logps/chosen": -1.3029353618621826,
+      "eval_logps/rejected": -1.4740725755691528,
+      "eval_loss": 1.652165174484253,
+      "eval_rewards/accuracies": 0.5675074458122253,
+      "eval_rewards/chosen": -1.3029353618621826,
+      "eval_rewards/margins": 0.17113718390464783,
+      "eval_rewards/rejected": -1.4740725755691528,
+      "eval_runtime": 40.6885,
+      "eval_samples_per_second": 33.056,
+      "eval_steps_per_second": 8.282,
+      "step": 2400
+    },
+    {
+      "epoch": 1.287171767854156,
+      "grad_norm": 7.870523096019233,
+      "learning_rate": 7.048185059920193e-07,
+      "logits/chosen": -0.11690235137939453,
+      "logits/rejected": 0.014140060171484947,
+      "logps/chosen": -1.2653751373291016,
+      "logps/rejected": -1.4744374752044678,
+      "loss": 1.6085,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.2653751373291016,
+      "rewards/margins": 0.20906221866607666,
+      "rewards/rejected": -1.4744374752044678,
+      "step": 2405
+    },
+    {
+      "epoch": 1.2898478006355578,
+      "grad_norm": 5.774238616199731,
+      "learning_rate": 7.033967763058516e-07,
+      "logits/chosen": -0.25922006368637085,
+      "logits/rejected": -0.08819141983985901,
+      "logps/chosen": -1.236149549484253,
+      "logps/rejected": -1.3286923170089722,
+      "loss": 1.6408,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.236149549484253,
+      "rewards/margins": 0.09254281967878342,
+      "rewards/rejected": -1.3286923170089722,
+      "step": 2410
+    },
+    {
+      "epoch": 1.2925238334169593,
+      "grad_norm": 6.588861507194845,
+      "learning_rate": 7.019730732632681e-07,
+      "logits/chosen": -0.08580441772937775,
+      "logits/rejected": -0.0363566055893898,
+      "logps/chosen": -1.1683250665664673,
+      "logps/rejected": -1.4436743259429932,
+      "loss": 1.5217,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.1683250665664673,
+      "rewards/margins": 0.2753491699695587,
+      "rewards/rejected": -1.4436743259429932,
+      "step": 2415
+    },
+    {
+      "epoch": 1.2951998661983608,
+      "grad_norm": 8.127051455874462,
+      "learning_rate": 7.005474106770418e-07,
+      "logits/chosen": -0.2234508991241455,
+      "logits/rejected": -0.12078411877155304,
+      "logps/chosen": -1.2782351970672607,
+      "logps/rejected": -1.5385297536849976,
+      "loss": 1.5969,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.2782351970672607,
+      "rewards/margins": 0.26029473543167114,
+      "rewards/rejected": -1.5385297536849976,
+      "step": 2420
+    },
+    {
+      "epoch": 1.2978758989797625,
+      "grad_norm": 8.007284967592312,
+      "learning_rate": 6.991198023789577e-07,
+      "logits/chosen": -0.11372745037078857,
+      "logits/rejected": -0.048462964594364166,
+      "logps/chosen": -1.2203960418701172,
+      "logps/rejected": -1.3983186483383179,
+      "loss": 1.5918,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.2203960418701172,
+      "rewards/margins": 0.1779225468635559,
+      "rewards/rejected": -1.3983186483383179,
+      "step": 2425
+    },
+    {
+      "epoch": 1.300551931761164,
+      "grad_norm": 8.464943925768683,
+      "learning_rate": 6.976902622196776e-07,
+      "logits/chosen": -0.11578784137964249,
+      "logits/rejected": -0.08799702674150467,
+      "logps/chosen": -1.3550939559936523,
+      "logps/rejected": -1.5142543315887451,
+      "loss": 1.6842,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.3550939559936523,
+      "rewards/margins": 0.1591602861881256,
+      "rewards/rejected": -1.5142543315887451,
+      "step": 2430
+    },
+    {
+      "epoch": 1.3032279645425655,
+      "grad_norm": 4.964557907334137,
+      "learning_rate": 6.962588040686064e-07,
+      "logits/chosen": -0.0680847316980362,
+      "logits/rejected": 0.029392996802926064,
+      "logps/chosen": -1.229901909828186,
+      "logps/rejected": -1.377708077430725,
+      "loss": 1.6048,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.229901909828186,
+      "rewards/margins": 0.14780613780021667,
+      "rewards/rejected": -1.377708077430725,
+      "step": 2435
+    },
+    {
+      "epoch": 1.3059039973239672,
+      "grad_norm": 9.173356236493861,
+      "learning_rate": 6.948254418137573e-07,
+      "logits/chosen": -0.22466211020946503,
+      "logits/rejected": -0.14043515920639038,
+      "logps/chosen": -1.208801507949829,
+      "logps/rejected": -1.401143193244934,
+      "loss": 1.5828,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.208801507949829,
+      "rewards/margins": 0.19234168529510498,
+      "rewards/rejected": -1.401143193244934,
+      "step": 2440
+    },
+    {
+      "epoch": 1.3085800301053687,
+      "grad_norm": 8.615344938569784,
+      "learning_rate": 6.933901893616174e-07,
+      "logits/chosen": -0.16824540495872498,
+      "logits/rejected": -0.04846107214689255,
+      "logps/chosen": -1.2559640407562256,
+      "logps/rejected": -1.3949588537216187,
+      "loss": 1.6227,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.2559640407562256,
+      "rewards/margins": 0.13899469375610352,
+      "rewards/rejected": -1.3949588537216187,
+      "step": 2445
+    },
+    {
+      "epoch": 1.3112560628867704,
+      "grad_norm": 6.534082551436406,
+      "learning_rate": 6.919530606370121e-07,
+      "logits/chosen": -0.16765077412128448,
+      "logits/rejected": -0.043173328042030334,
+      "logps/chosen": -1.2167965173721313,
+      "logps/rejected": -1.4837311506271362,
+      "loss": 1.5455,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.2167965173721313,
+      "rewards/margins": 0.2669346332550049,
+      "rewards/rejected": -1.4837311506271362,
+      "step": 2450
+    },
+    {
+      "epoch": 1.313932095668172,
+      "grad_norm": 5.162323626655467,
+      "learning_rate": 6.905140695829706e-07,
+      "logits/chosen": -0.18577834963798523,
+      "logits/rejected": -0.01245189644396305,
+      "logps/chosen": -1.3045991659164429,
+      "logps/rejected": -1.4210084676742554,
+      "loss": 1.6483,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.3045991659164429,
+      "rewards/margins": 0.11640922725200653,
+      "rewards/rejected": -1.4210084676742554,
+      "step": 2455
+    },
+    {
+      "epoch": 1.3166081284495736,
+      "grad_norm": 8.779305794595118,
+      "learning_rate": 6.890732301605904e-07,
+      "logits/chosen": -0.1385190486907959,
+      "logits/rejected": -0.06480330973863602,
+      "logps/chosen": -1.2952321767807007,
+      "logps/rejected": -1.3900924921035767,
+      "loss": 1.6514,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.2952321767807007,
+      "rewards/margins": 0.09486036002635956,
+      "rewards/rejected": -1.3900924921035767,
+      "step": 2460
+    },
+    {
+      "epoch": 1.3192841612309751,
+      "grad_norm": 6.879124186375867,
+      "learning_rate": 6.876305563489021e-07,
+      "logits/chosen": -0.1268390715122223,
+      "logits/rejected": -0.07661868631839752,
+      "logps/chosen": -1.2554209232330322,
+      "logps/rejected": -1.4585278034210205,
+      "loss": 1.6044,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.2554209232330322,
+      "rewards/margins": 0.20310668647289276,
+      "rewards/rejected": -1.4585278034210205,
+      "step": 2465
+    },
+    {
+      "epoch": 1.3219601940123766,
+      "grad_norm": 11.279106206347876,
+      "learning_rate": 6.861860621447331e-07,
+      "logits/chosen": -0.2476646453142166,
+      "logits/rejected": -0.14627087116241455,
+      "logps/chosen": -1.2362347841262817,
+      "logps/rejected": -1.330241084098816,
+      "loss": 1.6221,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.2362347841262817,
+      "rewards/margins": 0.09400641173124313,
+      "rewards/rejected": -1.330241084098816,
+      "step": 2470
+    },
+    {
+      "epoch": 1.3246362267937783,
+      "grad_norm": 6.601011703249647,
+      "learning_rate": 6.847397615625725e-07,
+      "logits/chosen": -0.17375004291534424,
+      "logits/rejected": -0.11787985265254974,
+      "logps/chosen": -1.2520265579223633,
+      "logps/rejected": -1.459517240524292,
+      "loss": 1.6086,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.2520265579223633,
+      "rewards/margins": 0.20749051868915558,
+      "rewards/rejected": -1.459517240524292,
+      "step": 2475
+    },
+    {
+      "epoch": 1.3273122595751798,
+      "grad_norm": 6.32496552850165,
+      "learning_rate": 6.83291668634435e-07,
+      "logits/chosen": -0.2697427272796631,
+      "logits/rejected": -0.1335819959640503,
+      "logps/chosen": -1.2836506366729736,
+      "logps/rejected": -1.494109869003296,
+      "loss": 1.6118,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.2836506366729736,
+      "rewards/margins": 0.21045920252799988,
+      "rewards/rejected": -1.494109869003296,
+      "step": 2480
+    },
+    {
+      "epoch": 1.3299882923565813,
+      "grad_norm": 7.993260328336925,
+      "learning_rate": 6.818417974097246e-07,
+      "logits/chosen": -0.11526402086019516,
+      "logits/rejected": 0.03870851546525955,
+      "logps/chosen": -1.2829654216766357,
+      "logps/rejected": -1.495354413986206,
+      "loss": 1.6461,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.2829654216766357,
+      "rewards/margins": 0.2123890370130539,
+      "rewards/rejected": -1.495354413986206,
+      "step": 2485
+    },
+    {
+      "epoch": 1.332664325137983,
+      "grad_norm": 7.319826411914482,
+      "learning_rate": 6.803901619550981e-07,
+      "logits/chosen": -0.210830956697464,
+      "logits/rejected": -0.1711214929819107,
+      "logps/chosen": -1.2724640369415283,
+      "logps/rejected": -1.5099023580551147,
+      "loss": 1.6111,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.2724640369415283,
+      "rewards/margins": 0.23743829131126404,
+      "rewards/rejected": -1.5099023580551147,
+      "step": 2490
+    },
+    {
+      "epoch": 1.3353403579193845,
+      "grad_norm": 7.088857844116769,
+      "learning_rate": 6.789367763543292e-07,
+      "logits/chosen": -0.12234888225793839,
+      "logits/rejected": -0.12407497316598892,
+      "logps/chosen": -1.2546168565750122,
+      "logps/rejected": -1.4399218559265137,
+      "loss": 1.6088,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.2546168565750122,
+      "rewards/margins": 0.18530511856079102,
+      "rewards/rejected": -1.4399218559265137,
+      "step": 2495
+    },
+    {
+      "epoch": 1.338016390700786,
+      "grad_norm": 7.917698914494699,
+      "learning_rate": 6.774816547081714e-07,
+      "logits/chosen": -0.10988526046276093,
+      "logits/rejected": 0.012320229783654213,
+      "logps/chosen": -1.2048064470291138,
+      "logps/rejected": -1.39044189453125,
+      "loss": 1.5679,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.2048064470291138,
+      "rewards/margins": 0.18563534319400787,
+      "rewards/rejected": -1.39044189453125,
+      "step": 2500
+    },
+    {
+      "epoch": 1.3406924234821878,
+      "grad_norm": 7.108609404496919,
+      "learning_rate": 6.760248111342211e-07,
+      "logits/chosen": -0.12231286615133286,
+      "logits/rejected": 0.017514532431960106,
+      "logps/chosen": -1.2140834331512451,
+      "logps/rejected": -1.38956618309021,
+      "loss": 1.575,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.2140834331512451,
+      "rewards/margins": 0.1754826456308365,
+      "rewards/rejected": -1.38956618309021,
+      "step": 2505
+    },
+    {
+      "epoch": 1.3433684562635893,
+      "grad_norm": 10.213421753778464,
+      "learning_rate": 6.745662597667813e-07,
+      "logits/chosen": -0.19724589586257935,
+      "logits/rejected": -0.09277534484863281,
+      "logps/chosen": -1.2082345485687256,
+      "logps/rejected": -1.4114068746566772,
+      "loss": 1.5714,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.2082345485687256,
+      "rewards/margins": 0.2031722366809845,
+      "rewards/rejected": -1.4114068746566772,
+      "step": 2510
+    },
+    {
+      "epoch": 1.3460444890449907,
+      "grad_norm": 6.255396425592991,
+      "learning_rate": 6.731060147567236e-07,
+      "logits/chosen": -0.12620656192302704,
+      "logits/rejected": -0.03647005930542946,
+      "logps/chosen": -1.291902780532837,
+      "logps/rejected": -1.401580572128296,
+      "loss": 1.6676,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.291902780532837,
+      "rewards/margins": 0.10967768728733063,
+      "rewards/rejected": -1.401580572128296,
+      "step": 2515
+    },
+    {
+      "epoch": 1.3487205218263925,
+      "grad_norm": 5.758726271944098,
+      "learning_rate": 6.716440902713515e-07,
+      "logits/chosen": -0.22279579937458038,
+      "logits/rejected": -0.16483859717845917,
+      "logps/chosen": -1.271669626235962,
+      "logps/rejected": -1.426734209060669,
+      "loss": 1.6181,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.271669626235962,
+      "rewards/margins": 0.1550646275281906,
+      "rewards/rejected": -1.426734209060669,
+      "step": 2520
+    },
+    {
+      "epoch": 1.351396554607794,
+      "grad_norm": 10.562665707587549,
+      "learning_rate": 6.701805004942627e-07,
+      "logits/chosen": -0.16978059709072113,
+      "logits/rejected": -0.11235268414020538,
+      "logps/chosen": -1.2969720363616943,
+      "logps/rejected": -1.4508230686187744,
+      "loss": 1.635,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.2969720363616943,
+      "rewards/margins": 0.1538509726524353,
+      "rewards/rejected": -1.4508230686187744,
+      "step": 2525
+    },
+    {
+      "epoch": 1.3540725873891954,
+      "grad_norm": 8.606204546975732,
+      "learning_rate": 6.687152596252119e-07,
+      "logits/chosen": -0.20366983115673065,
+      "logits/rejected": -0.1693367063999176,
+      "logps/chosen": -1.2421363592147827,
+      "logps/rejected": -1.4101701974868774,
+      "loss": 1.6156,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.2421363592147827,
+      "rewards/margins": 0.16803371906280518,
+      "rewards/rejected": -1.4101701974868774,
+      "step": 2530
+    },
+    {
+      "epoch": 1.3567486201705972,
+      "grad_norm": 6.1125997832319525,
+      "learning_rate": 6.672483818799722e-07,
+      "logits/chosen": -0.25032004714012146,
+      "logits/rejected": -0.14668934047222137,
+      "logps/chosen": -1.2616934776306152,
+      "logps/rejected": -1.4530704021453857,
+      "loss": 1.6074,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.2616934776306152,
+      "rewards/margins": 0.19137685000896454,
+      "rewards/rejected": -1.4530704021453857,
+      "step": 2535
+    },
+    {
+      "epoch": 1.3594246529519987,
+      "grad_norm": 9.518342328508894,
+      "learning_rate": 6.657798814901978e-07,
+      "logits/chosen": -0.16530966758728027,
+      "logits/rejected": -0.037598997354507446,
+      "logps/chosen": -1.3490521907806396,
+      "logps/rejected": -1.4344947338104248,
+      "loss": 1.6997,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.3490521907806396,
+      "rewards/margins": 0.0854424387216568,
+      "rewards/rejected": -1.4344947338104248,
+      "step": 2540
+    },
+    {
+      "epoch": 1.3621006857334002,
+      "grad_norm": 6.611910319793207,
+      "learning_rate": 6.643097727032863e-07,
+      "logits/chosen": -0.12749221920967102,
+      "logits/rejected": 0.005467116832733154,
+      "logps/chosen": -1.2553026676177979,
+      "logps/rejected": -1.4536714553833008,
+      "loss": 1.6155,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.2553026676177979,
+      "rewards/margins": 0.1983688771724701,
+      "rewards/rejected": -1.4536714553833008,
+      "step": 2545
+    },
+    {
+      "epoch": 1.3647767185148019,
+      "grad_norm": 6.999183872004218,
+      "learning_rate": 6.628380697822392e-07,
+      "logits/chosen": -0.15842144191265106,
+      "logits/rejected": -0.03417439013719559,
+      "logps/chosen": -1.2414013147354126,
+      "logps/rejected": -1.3254547119140625,
+      "loss": 1.6356,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.2414013147354126,
+      "rewards/margins": 0.08405350893735886,
+      "rewards/rejected": -1.3254547119140625,
+      "step": 2550
+    },
+    {
+      "epoch": 1.3674527512962034,
+      "grad_norm": 9.23276822558898,
+      "learning_rate": 6.61364787005525e-07,
+      "logits/chosen": -0.1209961324930191,
+      "logits/rejected": -0.06522391736507416,
+      "logps/chosen": -1.1821562051773071,
+      "logps/rejected": -1.4737653732299805,
+      "loss": 1.5397,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.1821562051773071,
+      "rewards/margins": 0.2916093170642853,
+      "rewards/rejected": -1.4737653732299805,
+      "step": 2555
+    },
+    {
+      "epoch": 1.3701287840776049,
+      "grad_norm": 7.419043651798917,
+      "learning_rate": 6.598899386669395e-07,
+      "logits/chosen": -0.11700377613306046,
+      "logits/rejected": -0.018622055649757385,
+      "logps/chosen": -1.2295806407928467,
+      "logps/rejected": -1.4211266040802002,
+      "loss": 1.5895,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.2295806407928467,
+      "rewards/margins": 0.19154588878154755,
+      "rewards/rejected": -1.4211266040802002,
+      "step": 2560
+    },
+    {
+      "epoch": 1.3728048168590066,
+      "grad_norm": 8.739827529012569,
+      "learning_rate": 6.584135390754679e-07,
+      "logits/chosen": -0.13102322816848755,
+      "logits/rejected": -0.0399283766746521,
+      "logps/chosen": -1.2161195278167725,
+      "logps/rejected": -1.4188120365142822,
+      "loss": 1.5784,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.2161195278167725,
+      "rewards/margins": 0.2026924192905426,
+      "rewards/rejected": -1.4188120365142822,
+      "step": 2565
+    },
+    {
+      "epoch": 1.375480849640408,
+      "grad_norm": 5.28190719308519,
+      "learning_rate": 6.569356025551454e-07,
+      "logits/chosen": -0.07822327315807343,
+      "logits/rejected": -0.028475111350417137,
+      "logps/chosen": -1.2194154262542725,
+      "logps/rejected": -1.4345922470092773,
+      "loss": 1.5667,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.2194154262542725,
+      "rewards/margins": 0.21517670154571533,
+      "rewards/rejected": -1.4345922470092773,
+      "step": 2570
+    },
+    {
+      "epoch": 1.3781568824218096,
+      "grad_norm": 9.348428251126803,
+      "learning_rate": 6.554561434449186e-07,
+      "logits/chosen": -0.222854346036911,
+      "logits/rejected": -0.1074608564376831,
+      "logps/chosen": -1.2142174243927002,
+      "logps/rejected": -1.444159984588623,
+      "loss": 1.5712,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.2142174243927002,
+      "rewards/margins": 0.22994264960289001,
+      "rewards/rejected": -1.444159984588623,
+      "step": 2575
+    },
+    {
+      "epoch": 1.3808329152032113,
+      "grad_norm": 10.517077556395618,
+      "learning_rate": 6.539751760985063e-07,
+      "logits/chosen": -0.12920764088630676,
+      "logits/rejected": -0.07156732678413391,
+      "logps/chosen": -1.314318299293518,
+      "logps/rejected": -1.4463088512420654,
+      "loss": 1.6786,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.314318299293518,
+      "rewards/margins": 0.13199035823345184,
+      "rewards/rejected": -1.4463088512420654,
+      "step": 2580
+    },
+    {
+      "epoch": 1.3835089479846128,
+      "grad_norm": 7.614676517974458,
+      "learning_rate": 6.524927148842602e-07,
+      "logits/chosen": -0.053183816373348236,
+      "logits/rejected": 0.08111951500177383,
+      "logps/chosen": -1.1608166694641113,
+      "logps/rejected": -1.417043685913086,
+      "loss": 1.544,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.1608166694641113,
+      "rewards/margins": 0.2562270164489746,
+      "rewards/rejected": -1.417043685913086,
+      "step": 2585
+    },
+    {
+      "epoch": 1.3861849807660143,
+      "grad_norm": 8.537552692374557,
+      "learning_rate": 6.510087741850254e-07,
+      "logits/chosen": -0.1578807532787323,
+      "logits/rejected": -0.03583088144659996,
+      "logps/chosen": -1.2086454629898071,
+      "logps/rejected": -1.418277382850647,
+      "loss": 1.5913,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.2086454629898071,
+      "rewards/margins": 0.20963187515735626,
+      "rewards/rejected": -1.418277382850647,
+      "step": 2590
+    },
+    {
+      "epoch": 1.388861013547416,
+      "grad_norm": 7.919291466775935,
+      "learning_rate": 6.495233683980012e-07,
+      "logits/chosen": -0.11748051643371582,
+      "logits/rejected": -0.0915294662117958,
+      "logps/chosen": -1.2231186628341675,
+      "logps/rejected": -1.4085519313812256,
+      "loss": 1.5786,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.2231186628341675,
+      "rewards/margins": 0.18543313443660736,
+      "rewards/rejected": -1.4085519313812256,
+      "step": 2595
+    },
+    {
+      "epoch": 1.3915370463288175,
+      "grad_norm": 9.665461917485846,
+      "learning_rate": 6.480365119346011e-07,
+      "logits/chosen": -0.04559020325541496,
+      "logits/rejected": 0.05628150701522827,
+      "logps/chosen": -1.2490603923797607,
+      "logps/rejected": -1.374648094177246,
+      "loss": 1.624,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.2490603923797607,
+      "rewards/margins": 0.12558765709400177,
+      "rewards/rejected": -1.374648094177246,
+      "step": 2600
+    },
+    {
+      "epoch": 1.394213079110219,
+      "grad_norm": 8.237334325510016,
+      "learning_rate": 6.465482192203129e-07,
+      "logits/chosen": -0.06613899767398834,
+      "logits/rejected": -0.03729189559817314,
+      "logps/chosen": -1.2390214204788208,
+      "logps/rejected": -1.3908802270889282,
+      "loss": 1.6125,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.2390214204788208,
+      "rewards/margins": 0.15185879170894623,
+      "rewards/rejected": -1.3908802270889282,
+      "step": 2605
+    },
+    {
+      "epoch": 1.3968891118916207,
+      "grad_norm": 9.157276767854839,
+      "learning_rate": 6.45058504694559e-07,
+      "logits/chosen": -0.04596097022294998,
+      "logits/rejected": 0.015534261241555214,
+      "logps/chosen": -1.2925312519073486,
+      "logps/rejected": -1.402549386024475,
+      "loss": 1.6832,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.2925312519073486,
+      "rewards/margins": 0.11001831293106079,
+      "rewards/rejected": -1.402549386024475,
+      "step": 2610
+    },
+    {
+      "epoch": 1.3995651446730222,
+      "grad_norm": 11.201303785821795,
+      "learning_rate": 6.435673828105564e-07,
+      "logits/chosen": -0.13810259103775024,
+      "logits/rejected": -0.02151239663362503,
+      "logps/chosen": -1.2091667652130127,
+      "logps/rejected": -1.4765537977218628,
+      "loss": 1.5709,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.2091667652130127,
+      "rewards/margins": 0.2673870921134949,
+      "rewards/rejected": -1.4765537977218628,
+      "step": 2615
+    },
+    {
+      "epoch": 1.402241177454424,
+      "grad_norm": 9.141856468133408,
+      "learning_rate": 6.420748680351763e-07,
+      "logits/chosen": -0.14668779075145721,
+      "logits/rejected": -0.15653502941131592,
+      "logps/chosen": -1.327497124671936,
+      "logps/rejected": -1.4073745012283325,
+      "loss": 1.7246,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -1.327497124671936,
+      "rewards/margins": 0.07987751811742783,
+      "rewards/rejected": -1.4073745012283325,
+      "step": 2620
+    },
+    {
+      "epoch": 1.4049172102358254,
+      "grad_norm": 8.592315234668606,
+      "learning_rate": 6.405809748488032e-07,
+      "logits/chosen": -0.09723938256502151,
+      "logits/rejected": 0.02080477401614189,
+      "logps/chosen": -1.2495981454849243,
+      "logps/rejected": -1.438523292541504,
+      "loss": 1.5908,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.2495981454849243,
+      "rewards/margins": 0.18892532587051392,
+      "rewards/rejected": -1.438523292541504,
+      "step": 2625
+    },
+    {
+      "epoch": 1.4075932430172269,
+      "grad_norm": 8.364445063548647,
+      "learning_rate": 6.390857177451956e-07,
+      "logits/chosen": -0.2485821545124054,
+      "logits/rejected": -0.07332058995962143,
+      "logps/chosen": -1.3173432350158691,
+      "logps/rejected": -1.4042927026748657,
+      "loss": 1.6781,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.3173432350158691,
+      "rewards/margins": 0.08694945275783539,
+      "rewards/rejected": -1.4042927026748657,
+      "step": 2630
+    },
+    {
+      "epoch": 1.4102692757986286,
+      "grad_norm": 10.206806565461111,
+      "learning_rate": 6.375891112313445e-07,
+      "logits/chosen": -0.12817701697349548,
+      "logits/rejected": -0.06272152066230774,
+      "logps/chosen": -1.2560198307037354,
+      "logps/rejected": -1.421544075012207,
+      "loss": 1.5929,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.2560198307037354,
+      "rewards/margins": 0.1655244678258896,
+      "rewards/rejected": -1.421544075012207,
+      "step": 2635
+    },
+    {
+      "epoch": 1.41294530858003,
+      "grad_norm": 6.646996513211527,
+      "learning_rate": 6.360911698273326e-07,
+      "logits/chosen": -0.08035455644130707,
+      "logits/rejected": -0.02586877904832363,
+      "logps/chosen": -1.3121914863586426,
+      "logps/rejected": -1.4780056476593018,
+      "loss": 1.6552,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.3121914863586426,
+      "rewards/margins": 0.1658143550157547,
+      "rewards/rejected": -1.4780056476593018,
+      "step": 2640
+    },
+    {
+      "epoch": 1.4156213413614318,
+      "grad_norm": 8.205700226774075,
+      "learning_rate": 6.345919080661944e-07,
+      "logits/chosen": -0.1574331670999527,
+      "logits/rejected": -0.09802021086215973,
+      "logps/chosen": -1.2450119256973267,
+      "logps/rejected": -1.4981917142868042,
+      "loss": 1.5773,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.2450119256973267,
+      "rewards/margins": 0.2531798481941223,
+      "rewards/rejected": -1.4981917142868042,
+      "step": 2645
+    },
+    {
+      "epoch": 1.4182973741428333,
+      "grad_norm": 6.74614958982811,
+      "learning_rate": 6.330913404937737e-07,
+      "logits/chosen": -0.20685835182666779,
+      "logits/rejected": -0.08519494533538818,
+      "logps/chosen": -1.2531399726867676,
+      "logps/rejected": -1.6164621114730835,
+      "loss": 1.6031,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.2531399726867676,
+      "rewards/margins": 0.3633221983909607,
+      "rewards/rejected": -1.6164621114730835,
+      "step": 2650
+    },
+    {
+      "epoch": 1.4209734069242348,
+      "grad_norm": 8.830012908631538,
+      "learning_rate": 6.315894816685838e-07,
+      "logits/chosen": -0.11997779458761215,
+      "logits/rejected": 0.0006568536045961082,
+      "logps/chosen": -1.158200740814209,
+      "logps/rejected": -1.3409008979797363,
+      "loss": 1.545,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.158200740814209,
+      "rewards/margins": 0.1827000081539154,
+      "rewards/rejected": -1.3409008979797363,
+      "step": 2655
+    },
+    {
+      "epoch": 1.4236494397056365,
+      "grad_norm": 8.649731098796604,
+      "learning_rate": 6.300863461616657e-07,
+      "logits/chosen": -0.07503405958414078,
+      "logits/rejected": -0.04890364408493042,
+      "logps/chosen": -1.1313199996948242,
+      "logps/rejected": -1.3792779445648193,
+      "loss": 1.5135,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.1313199996948242,
+      "rewards/margins": 0.24795803427696228,
+      "rewards/rejected": -1.3792779445648193,
+      "step": 2660
+    },
+    {
+      "epoch": 1.426325472487038,
+      "grad_norm": 7.4933448100013695,
+      "learning_rate": 6.285819485564465e-07,
+      "logits/chosen": -0.18574872612953186,
+      "logits/rejected": -0.09579996019601822,
+      "logps/chosen": -1.2786818742752075,
+      "logps/rejected": -1.4751293659210205,
+      "loss": 1.6158,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.2786818742752075,
+      "rewards/margins": 0.1964474618434906,
+      "rewards/rejected": -1.4751293659210205,
+      "step": 2665
+    },
+    {
+      "epoch": 1.4290015052684395,
+      "grad_norm": 8.864208288115423,
+      "learning_rate": 6.270763034485986e-07,
+      "logits/chosen": -0.09423649311065674,
+      "logits/rejected": -0.02666972018778324,
+      "logps/chosen": -1.3697813749313354,
+      "logps/rejected": -1.4894285202026367,
+      "loss": 1.7199,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.3697813749313354,
+      "rewards/margins": 0.1196470707654953,
+      "rewards/rejected": -1.4894285202026367,
+      "step": 2670
+    },
+    {
+      "epoch": 1.4316775380498412,
+      "grad_norm": 9.997330006904571,
+      "learning_rate": 6.255694254458972e-07,
+      "logits/chosen": -0.13650649785995483,
+      "logits/rejected": -0.005107997450977564,
+      "logps/chosen": -1.2864766120910645,
+      "logps/rejected": -1.4099823236465454,
+      "loss": 1.6435,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.2864766120910645,
+      "rewards/margins": 0.12350551784038544,
+      "rewards/rejected": -1.4099823236465454,
+      "step": 2675
+    },
+    {
+      "epoch": 1.4343535708312427,
+      "grad_norm": 9.257757138585266,
+      "learning_rate": 6.240613291680795e-07,
+      "logits/chosen": -0.19174785912036896,
+      "logits/rejected": -0.057482410222291946,
+      "logps/chosen": -1.2712281942367554,
+      "logps/rejected": -1.4052618741989136,
+      "loss": 1.642,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.2712281942367554,
+      "rewards/margins": 0.13403376936912537,
+      "rewards/rejected": -1.4052618741989136,
+      "step": 2680
+    },
+    {
+      "epoch": 1.4370296036126442,
+      "grad_norm": 9.12705530288166,
+      "learning_rate": 6.225520292467021e-07,
+      "logits/chosen": -0.13390594720840454,
+      "logits/rejected": 0.035317469388246536,
+      "logps/chosen": -1.26984703540802,
+      "logps/rejected": -1.3742210865020752,
+      "loss": 1.6372,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.26984703540802,
+      "rewards/margins": 0.10437413305044174,
+      "rewards/rejected": -1.3742210865020752,
+      "step": 2685
+    },
+    {
+      "epoch": 1.439705636394046,
+      "grad_norm": 10.132403967111959,
+      "learning_rate": 6.210415403249993e-07,
+      "logits/chosen": -0.27369385957717896,
+      "logits/rejected": -0.082986980676651,
+      "logps/chosen": -1.2638477087020874,
+      "logps/rejected": -1.5354796648025513,
+      "loss": 1.6034,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.2638477087020874,
+      "rewards/margins": 0.27163180708885193,
+      "rewards/rejected": -1.5354796648025513,
+      "step": 2690
+    },
+    {
+      "epoch": 1.4423816691754474,
+      "grad_norm": 6.391142485683344,
+      "learning_rate": 6.195298770577415e-07,
+      "logits/chosen": -0.14904870092868805,
+      "logits/rejected": -0.13162747025489807,
+      "logps/chosen": -1.2408156394958496,
+      "logps/rejected": -1.454119324684143,
+      "loss": 1.5791,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.2408156394958496,
+      "rewards/margins": 0.21330364048480988,
+      "rewards/rejected": -1.454119324684143,
+      "step": 2695
+    },
+    {
+      "epoch": 1.445057701956849,
+      "grad_norm": 7.727114200402468,
+      "learning_rate": 6.180170541110923e-07,
+      "logits/chosen": -0.18305456638336182,
+      "logits/rejected": -0.02566959336400032,
+      "logps/chosen": -1.2899725437164307,
+      "logps/rejected": -1.4568368196487427,
+      "loss": 1.6393,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.2899725437164307,
+      "rewards/margins": 0.16686417162418365,
+      "rewards/rejected": -1.4568368196487427,
+      "step": 2700
+    },
+    {
+      "epoch": 1.4477337347382506,
+      "grad_norm": 5.701958769249109,
+      "learning_rate": 6.165030861624663e-07,
+      "logits/chosen": -0.19134236872196198,
+      "logits/rejected": -0.025602400302886963,
+      "logps/chosen": -1.1669811010360718,
+      "logps/rejected": -1.4505869150161743,
+      "loss": 1.5034,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.1669811010360718,
+      "rewards/margins": 0.2836058437824249,
+      "rewards/rejected": -1.4505869150161743,
+      "step": 2705
+    },
+    {
+      "epoch": 1.4504097675196521,
+      "grad_norm": 7.643404876619234,
+      "learning_rate": 6.149879879003876e-07,
+      "logits/chosen": -0.10536251217126846,
+      "logits/rejected": -0.08227679133415222,
+      "logps/chosen": -1.2573299407958984,
+      "logps/rejected": -1.473145604133606,
+      "loss": 1.5871,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.2573299407958984,
+      "rewards/margins": 0.21581566333770752,
+      "rewards/rejected": -1.473145604133606,
+      "step": 2710
+    },
+    {
+      "epoch": 1.4530858003010536,
+      "grad_norm": 7.684216778403979,
+      "learning_rate": 6.13471774024346e-07,
+      "logits/chosen": -0.23534438014030457,
+      "logits/rejected": -0.16144892573356628,
+      "logps/chosen": -1.1998146772384644,
+      "logps/rejected": -1.3855111598968506,
+      "loss": 1.5734,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.1998146772384644,
+      "rewards/margins": 0.18569636344909668,
+      "rewards/rejected": -1.3855111598968506,
+      "step": 2715
+    },
+    {
+      "epoch": 1.4557618330824553,
+      "grad_norm": 7.474698672929521,
+      "learning_rate": 6.119544592446551e-07,
+      "logits/chosen": -0.18152351677417755,
+      "logits/rejected": -0.08897839486598969,
+      "logps/chosen": -1.2277088165283203,
+      "logps/rejected": -1.3388454914093018,
+      "loss": 1.6011,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.2277088165283203,
+      "rewards/margins": 0.11113659292459488,
+      "rewards/rejected": -1.3388454914093018,
+      "step": 2720
+    },
+    {
+      "epoch": 1.4584378658638568,
+      "grad_norm": 9.308831518397065,
+      "learning_rate": 6.104360582823096e-07,
+      "logits/chosen": -0.1577589064836502,
+      "logits/rejected": -0.06443767249584198,
+      "logps/chosen": -1.2389967441558838,
+      "logps/rejected": -1.4576349258422852,
+      "loss": 1.6043,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.2389967441558838,
+      "rewards/margins": 0.21863818168640137,
+      "rewards/rejected": -1.4576349258422852,
+      "step": 2725
+    },
+    {
+      "epoch": 1.4611138986452583,
+      "grad_norm": 8.364378865528284,
+      "learning_rate": 6.089165858688423e-07,
+      "logits/chosen": -0.19063672423362732,
+      "logits/rejected": -0.055591851472854614,
+      "logps/chosen": -1.239959955215454,
+      "logps/rejected": -1.4646373987197876,
+      "loss": 1.6223,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.239959955215454,
+      "rewards/margins": 0.22467747330665588,
+      "rewards/rejected": -1.4646373987197876,
+      "step": 2730
+    },
+    {
+      "epoch": 1.46378993142666,
+      "grad_norm": 7.2502968807995165,
+      "learning_rate": 6.073960567461811e-07,
+      "logits/chosen": -0.15855631232261658,
+      "logits/rejected": 0.005321502685546875,
+      "logps/chosen": -1.154059648513794,
+      "logps/rejected": -1.4797316789627075,
+      "loss": 1.5238,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.154059648513794,
+      "rewards/margins": 0.32567209005355835,
+      "rewards/rejected": -1.4797316789627075,
+      "step": 2735
+    },
+    {
+      "epoch": 1.4664659642080615,
+      "grad_norm": 9.284400020312043,
+      "learning_rate": 6.058744856665065e-07,
+      "logits/chosen": -0.1830758899450302,
+      "logits/rejected": -0.09102828800678253,
+      "logps/chosen": -1.192286491394043,
+      "logps/rejected": -1.4547383785247803,
+      "loss": 1.5673,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.192286491394043,
+      "rewards/margins": 0.2624519467353821,
+      "rewards/rejected": -1.4547383785247803,
+      "step": 2740
+    },
+    {
+      "epoch": 1.469141996989463,
+      "grad_norm": 7.236813918750393,
+      "learning_rate": 6.043518873921074e-07,
+      "logits/chosen": -0.1663316786289215,
+      "logits/rejected": -0.07601561397314072,
+      "logps/chosen": -1.2092053890228271,
+      "logps/rejected": -1.3376567363739014,
+      "loss": 1.5912,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.2092053890228271,
+      "rewards/margins": 0.12845143675804138,
+      "rewards/rejected": -1.3376567363739014,
+      "step": 2745
+    },
+    {
+      "epoch": 1.4718180297708647,
+      "grad_norm": 7.953164084744921,
+      "learning_rate": 6.028282766952393e-07,
+      "logits/chosen": -0.13207189738750458,
+      "logits/rejected": -0.05418176203966141,
+      "logps/chosen": -1.2845780849456787,
+      "logps/rejected": -1.52390718460083,
+      "loss": 1.6049,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.2845780849456787,
+      "rewards/margins": 0.2393290102481842,
+      "rewards/rejected": -1.52390718460083,
+      "step": 2750
+    },
+    {
+      "epoch": 1.4744940625522662,
+      "grad_norm": 9.956779025794944,
+      "learning_rate": 6.013036683579798e-07,
+      "logits/chosen": -0.08539600670337677,
+      "logits/rejected": 0.03946538269519806,
+      "logps/chosen": -1.2316306829452515,
+      "logps/rejected": -1.3691304922103882,
+      "loss": 1.5971,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.2316306829452515,
+      "rewards/margins": 0.13749974966049194,
+      "rewards/rejected": -1.3691304922103882,
+      "step": 2755
+    },
+    {
+      "epoch": 1.4771700953336677,
+      "grad_norm": 7.9294848838307015,
+      "learning_rate": 5.997780771720854e-07,
+      "logits/chosen": -0.22258487343788147,
+      "logits/rejected": -0.08026249706745148,
+      "logps/chosen": -1.2677894830703735,
+      "logps/rejected": -1.5182039737701416,
+      "loss": 1.6203,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.2677894830703735,
+      "rewards/margins": 0.25041455030441284,
+      "rewards/rejected": -1.5182039737701416,
+      "step": 2760
+    },
+    {
+      "epoch": 1.4798461281150694,
+      "grad_norm": 8.259625818318181,
+      "learning_rate": 5.982515179388486e-07,
+      "logits/chosen": -0.12650857865810394,
+      "logits/rejected": -0.011118963360786438,
+      "logps/chosen": -1.188811182975769,
+      "logps/rejected": -1.3890987634658813,
+      "loss": 1.5566,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.188811182975769,
+      "rewards/margins": 0.20028741657733917,
+      "rewards/rejected": -1.3890987634658813,
+      "step": 2765
+    },
+    {
+      "epoch": 1.482522160896471,
+      "grad_norm": 7.953137807396655,
+      "learning_rate": 5.967240054689541e-07,
+      "logits/chosen": -0.16903971135616302,
+      "logits/rejected": -0.11257338523864746,
+      "logps/chosen": -1.218370795249939,
+      "logps/rejected": -1.2844295501708984,
+      "loss": 1.6292,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.218370795249939,
+      "rewards/margins": 0.06605866551399231,
+      "rewards/rejected": -1.2844295501708984,
+      "step": 2770
+    },
+    {
+      "epoch": 1.4851981936778724,
+      "grad_norm": 5.5455741309036615,
+      "learning_rate": 5.951955545823342e-07,
+      "logits/chosen": -0.14832541346549988,
+      "logits/rejected": -0.13577041029930115,
+      "logps/chosen": -1.1799458265304565,
+      "logps/rejected": -1.3889585733413696,
+      "loss": 1.5523,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.1799458265304565,
+      "rewards/margins": 0.20901262760162354,
+      "rewards/rejected": -1.3889585733413696,
+      "step": 2775
+    },
+    {
+      "epoch": 1.4878742264592741,
+      "grad_norm": 5.398542683932957,
+      "learning_rate": 5.936661801080263e-07,
+      "logits/chosen": -0.13961201906204224,
+      "logits/rejected": -0.0569029338657856,
+      "logps/chosen": -1.3484827280044556,
+      "logps/rejected": -1.4868760108947754,
+      "loss": 1.6867,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.3484827280044556,
+      "rewards/margins": 0.1383933126926422,
+      "rewards/rejected": -1.4868760108947754,
+      "step": 2780
+    },
+    {
+      "epoch": 1.4905502592406756,
+      "grad_norm": 7.595821185890207,
+      "learning_rate": 5.92135896884028e-07,
+      "logits/chosen": -0.1922890841960907,
+      "logits/rejected": -0.0873006209731102,
+      "logps/chosen": -1.3084876537322998,
+      "logps/rejected": -1.4598156213760376,
+      "loss": 1.6707,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.3084876537322998,
+      "rewards/margins": 0.1513279229402542,
+      "rewards/rejected": -1.4598156213760376,
+      "step": 2785
+    },
+    {
+      "epoch": 1.4932262920220774,
+      "grad_norm": 11.569421426806633,
+      "learning_rate": 5.906047197571541e-07,
+      "logits/chosen": -0.11923675239086151,
+      "logits/rejected": -0.13315056264400482,
+      "logps/chosen": -1.2030599117279053,
+      "logps/rejected": -1.418312907218933,
+      "loss": 1.5777,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.2030599117279053,
+      "rewards/margins": 0.21525295078754425,
+      "rewards/rejected": -1.418312907218933,
+      "step": 2790
+    },
+    {
+      "epoch": 1.4959023248034788,
+      "grad_norm": 7.132297547112176,
+      "learning_rate": 5.890726635828919e-07,
+      "logits/chosen": -0.02672731503844261,
+      "logits/rejected": -0.013998660258948803,
+      "logps/chosen": -1.1751325130462646,
+      "logps/rejected": -1.3113696575164795,
+      "loss": 1.6077,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.1751325130462646,
+      "rewards/margins": 0.13623718917369843,
+      "rewards/rejected": -1.3113696575164795,
+      "step": 2795
+    },
+    {
+      "epoch": 1.4985783575848803,
+      "grad_norm": 7.321205535658056,
+      "learning_rate": 5.875397432252569e-07,
+      "logits/chosen": -0.20826506614685059,
+      "logits/rejected": -0.13590756058692932,
+      "logps/chosen": -1.319227933883667,
+      "logps/rejected": -1.5354816913604736,
+      "loss": 1.6687,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.319227933883667,
+      "rewards/margins": 0.21625368297100067,
+      "rewards/rejected": -1.5354816913604736,
+      "step": 2800
+    },
+    {
+      "epoch": 1.4985783575848803,
+      "eval_logits/chosen": 0.10514389723539352,
+      "eval_logits/rejected": 0.17535388469696045,
+      "eval_logps/chosen": -1.299961805343628,
+      "eval_logps/rejected": -1.4737409353256226,
+      "eval_loss": 1.6487802267074585,
+      "eval_rewards/accuracies": 0.5697329640388489,
+      "eval_rewards/chosen": -1.299961805343628,
+      "eval_rewards/margins": 0.1737791746854782,
+      "eval_rewards/rejected": -1.4737409353256226,
+      "eval_runtime": 40.7375,
+      "eval_samples_per_second": 33.016,
+      "eval_steps_per_second": 8.272,
+      "step": 2800
+    },
+    {
+      "epoch": 1.5012543903662818,
+      "grad_norm": 5.001064596005135,
+      "learning_rate": 5.860059735566491e-07,
+      "logits/chosen": -0.2607018053531647,
+      "logits/rejected": -0.1325281709432602,
+      "logps/chosen": -1.1559540033340454,
+      "logps/rejected": -1.3943657875061035,
+      "loss": 1.531,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.1559540033340454,
+      "rewards/margins": 0.23841187357902527,
+      "rewards/rejected": -1.3943657875061035,
+      "step": 2805
+    },
+    {
+      "epoch": 1.5039304231476835,
+      "grad_norm": 12.124841820595227,
+      "learning_rate": 5.844713694577087e-07,
+      "logits/chosen": -0.16709429025650024,
+      "logits/rejected": -0.15542718768119812,
+      "logps/chosen": -1.2861963510513306,
+      "logps/rejected": -1.4874119758605957,
+      "loss": 1.6393,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.2861963510513306,
+      "rewards/margins": 0.20121553540229797,
+      "rewards/rejected": -1.4874119758605957,
+      "step": 2810
+    },
+    {
+      "epoch": 1.5066064559290853,
+      "grad_norm": 7.814378815191505,
+      "learning_rate": 5.829359458171714e-07,
+      "logits/chosen": -0.11754331737756729,
+      "logits/rejected": -0.010437699034810066,
+      "logps/chosen": -1.280771255493164,
+      "logps/rejected": -1.4639467000961304,
+      "loss": 1.612,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.280771255493164,
+      "rewards/margins": 0.1831754595041275,
+      "rewards/rejected": -1.4639467000961304,
+      "step": 2815
+    },
+    {
+      "epoch": 1.5092824887104868,
+      "grad_norm": 6.59949921623073,
+      "learning_rate": 5.81399717531724e-07,
+      "logits/chosen": -0.11054511368274689,
+      "logits/rejected": 0.01979086734354496,
+      "logps/chosen": -1.2499945163726807,
+      "logps/rejected": -1.3047266006469727,
+      "loss": 1.6465,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.2499945163726807,
+      "rewards/margins": 0.05473214387893677,
+      "rewards/rejected": -1.3047266006469727,
+      "step": 2820
+    },
+    {
+      "epoch": 1.5119585214918883,
+      "grad_norm": 7.0873516438434265,
+      "learning_rate": 5.798626995058602e-07,
+      "logits/chosen": -0.1889365315437317,
+      "logits/rejected": -0.0532827265560627,
+      "logps/chosen": -1.3098100423812866,
+      "logps/rejected": -1.563604474067688,
+      "loss": 1.6327,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.3098100423812866,
+      "rewards/margins": 0.2537943720817566,
+      "rewards/rejected": -1.563604474067688,
+      "step": 2825
+    },
+    {
+      "epoch": 1.51463455427329,
+      "grad_norm": 7.1073016737017705,
+      "learning_rate": 5.783249066517354e-07,
+      "logits/chosen": -0.1716947853565216,
+      "logits/rejected": -0.04569972679018974,
+      "logps/chosen": -1.309134602546692,
+      "logps/rejected": -1.3283313512802124,
+      "loss": 1.7066,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -1.309134602546692,
+      "rewards/margins": 0.019196750596165657,
+      "rewards/rejected": -1.3283313512802124,
+      "step": 2830
+    },
+    {
+      "epoch": 1.5173105870546915,
+      "grad_norm": 9.573012875878954,
+      "learning_rate": 5.767863538890228e-07,
+      "logits/chosen": -0.16613741219043732,
+      "logits/rejected": -0.031196704134345055,
+      "logps/chosen": -1.2335695028305054,
+      "logps/rejected": -1.385694980621338,
+      "loss": 1.6026,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.2335695028305054,
+      "rewards/margins": 0.15212538838386536,
+      "rewards/rejected": -1.385694980621338,
+      "step": 2835
+    },
+    {
+      "epoch": 1.519986619836093,
+      "grad_norm": 8.765137445446333,
+      "learning_rate": 5.75247056144768e-07,
+      "logits/chosen": -0.14558474719524384,
+      "logits/rejected": -0.054780591279268265,
+      "logps/chosen": -1.2625131607055664,
+      "logps/rejected": -1.4063299894332886,
+      "loss": 1.6113,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.2625131607055664,
+      "rewards/margins": 0.14381691813468933,
+      "rewards/rejected": -1.4063299894332886,
+      "step": 2840
+    },
+    {
+      "epoch": 1.5226626526174947,
+      "grad_norm": 7.848634771449643,
+      "learning_rate": 5.737070283532444e-07,
+      "logits/chosen": -0.13879218697547913,
+      "logits/rejected": -0.07548417150974274,
+      "logps/chosen": -1.2000709772109985,
+      "logps/rejected": -1.4368489980697632,
+      "loss": 1.5687,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.2000709772109985,
+      "rewards/margins": 0.23677799105644226,
+      "rewards/rejected": -1.4368489980697632,
+      "step": 2845
+    },
+    {
+      "epoch": 1.5253386853988962,
+      "grad_norm": 9.429682825684978,
+      "learning_rate": 5.721662854558084e-07,
+      "logits/chosen": -0.18743689358234406,
+      "logits/rejected": -0.1311352550983429,
+      "logps/chosen": -1.2835838794708252,
+      "logps/rejected": -1.4209805727005005,
+      "loss": 1.6482,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -1.2835838794708252,
+      "rewards/margins": 0.13739685714244843,
+      "rewards/rejected": -1.4209805727005005,
+      "step": 2850
+    },
+    {
+      "epoch": 1.5280147181802977,
+      "grad_norm": 6.648982314070762,
+      "learning_rate": 5.706248424007545e-07,
+      "logits/chosen": -0.2049282044172287,
+      "logits/rejected": -0.07677256315946579,
+      "logps/chosen": -1.3301162719726562,
+      "logps/rejected": -1.5274568796157837,
+      "loss": 1.6551,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.3301162719726562,
+      "rewards/margins": 0.1973404586315155,
+      "rewards/rejected": -1.5274568796157837,
+      "step": 2855
+    },
+    {
+      "epoch": 1.5306907509616994,
+      "grad_norm": 8.789640159262424,
+      "learning_rate": 5.690827141431699e-07,
+      "logits/chosen": -0.2506573796272278,
+      "logits/rejected": -0.11205937713384628,
+      "logps/chosen": -1.218569040298462,
+      "logps/rejected": -1.350964903831482,
+      "loss": 1.5879,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.218569040298462,
+      "rewards/margins": 0.13239574432373047,
+      "rewards/rejected": -1.350964903831482,
+      "step": 2860
+    },
+    {
+      "epoch": 1.5333667837431009,
+      "grad_norm": 7.910421340948889,
+      "learning_rate": 5.675399156447897e-07,
+      "logits/chosen": -0.299457848072052,
+      "logits/rejected": -0.19384565949440002,
+      "logps/chosen": -1.2221951484680176,
+      "logps/rejected": -1.4229544401168823,
+      "loss": 1.6032,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.2221951484680176,
+      "rewards/margins": 0.20075933635234833,
+      "rewards/rejected": -1.4229544401168823,
+      "step": 2865
+    },
+    {
+      "epoch": 1.5360428165245024,
+      "grad_norm": 9.374208664023072,
+      "learning_rate": 5.659964618738515e-07,
+      "logits/chosen": -0.1947907656431198,
+      "logits/rejected": -0.1025640144944191,
+      "logps/chosen": -1.2786771059036255,
+      "logps/rejected": -1.3355814218521118,
+      "loss": 1.6788,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -1.2786771059036255,
+      "rewards/margins": 0.05690423399209976,
+      "rewards/rejected": -1.3355814218521118,
+      "step": 2870
+    },
+    {
+      "epoch": 1.538718849305904,
+      "grad_norm": 8.115924306598316,
+      "learning_rate": 5.644523678049509e-07,
+      "logits/chosen": -0.1951065957546234,
+      "logits/rejected": -0.1314360648393631,
+      "logps/chosen": -1.2781469821929932,
+      "logps/rejected": -1.3914257287979126,
+      "loss": 1.6343,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.2781469821929932,
+      "rewards/margins": 0.11327870935201645,
+      "rewards/rejected": -1.3914257287979126,
+      "step": 2875
+    },
+    {
+      "epoch": 1.5413948820873056,
+      "grad_norm": 8.892601728738727,
+      "learning_rate": 5.629076484188952e-07,
+      "logits/chosen": -0.07616610080003738,
+      "logits/rejected": 0.016429344192147255,
+      "logps/chosen": -1.2245382070541382,
+      "logps/rejected": -1.440077781677246,
+      "loss": 1.6103,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.2245382070541382,
+      "rewards/margins": 0.21553964912891388,
+      "rewards/rejected": -1.440077781677246,
+      "step": 2880
+    },
+    {
+      "epoch": 1.544070914868707,
+      "grad_norm": 8.237630831845575,
+      "learning_rate": 5.613623187025587e-07,
+      "logits/chosen": -0.15724192559719086,
+      "logits/rejected": -0.05733314901590347,
+      "logps/chosen": -1.2409113645553589,
+      "logps/rejected": -1.400888442993164,
+      "loss": 1.6074,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.2409113645553589,
+      "rewards/margins": 0.1599770486354828,
+      "rewards/rejected": -1.400888442993164,
+      "step": 2885
+    },
+    {
+      "epoch": 1.5467469476501088,
+      "grad_norm": 7.162494464066858,
+      "learning_rate": 5.598163936487369e-07,
+      "logits/chosen": -0.2565734386444092,
+      "logits/rejected": -0.10567812621593475,
+      "logps/chosen": -1.2417627573013306,
+      "logps/rejected": -1.4008761644363403,
+      "loss": 1.6039,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.2417627573013306,
+      "rewards/margins": 0.15911336243152618,
+      "rewards/rejected": -1.4008761644363403,
+      "step": 2890
+    },
+    {
+      "epoch": 1.5494229804315103,
+      "grad_norm": 7.713818723912763,
+      "learning_rate": 5.582698882560017e-07,
+      "logits/chosen": -0.24017128348350525,
+      "logits/rejected": -0.13441559672355652,
+      "logps/chosen": -1.1790987253189087,
+      "logps/rejected": -1.3518292903900146,
+      "loss": 1.5616,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.1790987253189087,
+      "rewards/margins": 0.17273077368736267,
+      "rewards/rejected": -1.3518292903900146,
+      "step": 2895
+    },
+    {
+      "epoch": 1.5520990132129118,
+      "grad_norm": 7.728791758359173,
+      "learning_rate": 5.567228175285549e-07,
+      "logits/chosen": -0.14355513453483582,
+      "logits/rejected": -0.06344401091337204,
+      "logps/chosen": -1.2629728317260742,
+      "logps/rejected": -1.4144147634506226,
+      "loss": 1.6173,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.2629728317260742,
+      "rewards/margins": 0.15144187211990356,
+      "rewards/rejected": -1.4144147634506226,
+      "step": 2900
+    },
+    {
+      "epoch": 1.5547750459943135,
+      "grad_norm": 6.08821972601008,
+      "learning_rate": 5.551751964760838e-07,
+      "logits/chosen": -0.09214814752340317,
+      "logits/rejected": -0.068619005382061,
+      "logps/chosen": -1.22005295753479,
+      "logps/rejected": -1.4123097658157349,
+      "loss": 1.5815,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.22005295753479,
+      "rewards/margins": 0.19225691258907318,
+      "rewards/rejected": -1.4123097658157349,
+      "step": 2905
+    },
+    {
+      "epoch": 1.557451078775715,
+      "grad_norm": 8.049352936261435,
+      "learning_rate": 5.536270401136145e-07,
+      "logits/chosen": -0.15920159220695496,
+      "logits/rejected": -0.1046360582113266,
+      "logps/chosen": -1.2048557996749878,
+      "logps/rejected": -1.360347032546997,
+      "loss": 1.5893,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.2048557996749878,
+      "rewards/margins": 0.15549114346504211,
+      "rewards/rejected": -1.360347032546997,
+      "step": 2910
+    },
+    {
+      "epoch": 1.5601271115571165,
+      "grad_norm": 9.62263310263903,
+      "learning_rate": 5.520783634613667e-07,
+      "logits/chosen": -0.10347437858581543,
+      "logits/rejected": 0.02849293313920498,
+      "logps/chosen": -1.2959470748901367,
+      "logps/rejected": -1.4177794456481934,
+      "loss": 1.6532,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.2959470748901367,
+      "rewards/margins": 0.12183227390050888,
+      "rewards/rejected": -1.4177794456481934,
+      "step": 2915
+    },
+    {
+      "epoch": 1.5628031443385182,
+      "grad_norm": 5.957591983364518,
+      "learning_rate": 5.505291815446082e-07,
+      "logits/chosen": -0.09513330459594727,
+      "logits/rejected": -0.00013175010099075735,
+      "logps/chosen": -1.2686645984649658,
+      "logps/rejected": -1.4559599161148071,
+      "loss": 1.6123,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.2686645984649658,
+      "rewards/margins": 0.18729540705680847,
+      "rewards/rejected": -1.4559599161148071,
+      "step": 2920
+    },
+    {
+      "epoch": 1.5654791771199197,
+      "grad_norm": 8.750103612028086,
+      "learning_rate": 5.489795093935089e-07,
+      "logits/chosen": -0.10519526898860931,
+      "logits/rejected": -0.07157649099826813,
+      "logps/chosen": -1.1874984502792358,
+      "logps/rejected": -1.4648293256759644,
+      "loss": 1.5577,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.1874984502792358,
+      "rewards/margins": 0.2773308753967285,
+      "rewards/rejected": -1.4648293256759644,
+      "step": 2925
+    },
+    {
+      "epoch": 1.5681552099013212,
+      "grad_norm": 10.277760462873392,
+      "learning_rate": 5.474293620429946e-07,
+      "logits/chosen": -0.22603020071983337,
+      "logits/rejected": -0.09562810510396957,
+      "logps/chosen": -1.2209093570709229,
+      "logps/rejected": -1.4970707893371582,
+      "loss": 1.5691,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.2209093570709229,
+      "rewards/margins": 0.276161253452301,
+      "rewards/rejected": -1.4970707893371582,
+      "step": 2930
+    },
+    {
+      "epoch": 1.570831242682723,
+      "grad_norm": 7.6856069730391345,
+      "learning_rate": 5.458787545326018e-07,
+      "logits/chosen": -0.18429319560527802,
+      "logits/rejected": -0.06572946906089783,
+      "logps/chosen": -1.2605324983596802,
+      "logps/rejected": -1.4396363496780396,
+      "loss": 1.6223,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.2605324983596802,
+      "rewards/margins": 0.17910373210906982,
+      "rewards/rejected": -1.4396363496780396,
+      "step": 2935
+    },
+    {
+      "epoch": 1.5735072754641244,
+      "grad_norm": 6.231542616149614,
+      "learning_rate": 5.443277019063311e-07,
+      "logits/chosen": -0.18334154784679413,
+      "logits/rejected": -0.044742949306964874,
+      "logps/chosen": -1.2537963390350342,
+      "logps/rejected": -1.4843844175338745,
+      "loss": 1.6136,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.2537963390350342,
+      "rewards/margins": 0.23058822751045227,
+      "rewards/rejected": -1.4843844175338745,
+      "step": 2940
+    },
+    {
+      "epoch": 1.5761833082455259,
+      "grad_norm": 9.884251728039047,
+      "learning_rate": 5.427762192125023e-07,
+      "logits/chosen": -0.2172185480594635,
+      "logits/rejected": -0.10349267721176147,
+      "logps/chosen": -1.2288261651992798,
+      "logps/rejected": -1.3491674661636353,
+      "loss": 1.6158,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.2288261651992798,
+      "rewards/margins": 0.12034142017364502,
+      "rewards/rejected": -1.3491674661636353,
+      "step": 2945
+    },
+    {
+      "epoch": 1.5788593410269276,
+      "grad_norm": 9.984560793832488,
+      "learning_rate": 5.41224321503607e-07,
+      "logits/chosen": -0.08993600308895111,
+      "logits/rejected": 0.1004088744521141,
+      "logps/chosen": -1.198878526687622,
+      "logps/rejected": -1.4119997024536133,
+      "loss": 1.5484,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.198878526687622,
+      "rewards/margins": 0.21312110126018524,
+      "rewards/rejected": -1.4119997024536133,
+      "step": 2950
+    },
+    {
+      "epoch": 1.5815353738083293,
+      "grad_norm": 7.432126220058146,
+      "learning_rate": 5.396720238361637e-07,
+      "logits/chosen": -0.10413719713687897,
+      "logits/rejected": -0.02635631524026394,
+      "logps/chosen": -1.2171152830123901,
+      "logps/rejected": -1.4342138767242432,
+      "loss": 1.5748,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.2171152830123901,
+      "rewards/margins": 0.21709854900836945,
+      "rewards/rejected": -1.4342138767242432,
+      "step": 2955
+    },
+    {
+      "epoch": 1.5842114065897306,
+      "grad_norm": 8.632437936611266,
+      "learning_rate": 5.381193412705711e-07,
+      "logits/chosen": -0.21219193935394287,
+      "logits/rejected": -0.11716421693563461,
+      "logps/chosen": -1.2380011081695557,
+      "logps/rejected": -1.4030927419662476,
+      "loss": 1.5898,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.2380011081695557,
+      "rewards/margins": 0.16509170830249786,
+      "rewards/rejected": -1.4030927419662476,
+      "step": 2960
+    },
+    {
+      "epoch": 1.5868874393711323,
+      "grad_norm": 11.416226627501322,
+      "learning_rate": 5.365662888709622e-07,
+      "logits/chosen": -0.16225700080394745,
+      "logits/rejected": -0.09003710001707077,
+      "logps/chosen": -1.2026822566986084,
+      "logps/rejected": -1.3658852577209473,
+      "loss": 1.58,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.2026822566986084,
+      "rewards/margins": 0.1632029116153717,
+      "rewards/rejected": -1.3658852577209473,
+      "step": 2965
+    },
+    {
+      "epoch": 1.589563472152534,
+      "grad_norm": 11.522106023505476,
+      "learning_rate": 5.350128817050585e-07,
+      "logits/chosen": -0.16338010132312775,
+      "logits/rejected": -0.026224765926599503,
+      "logps/chosen": -1.2979391813278198,
+      "logps/rejected": -1.40617036819458,
+      "loss": 1.6787,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.2979391813278198,
+      "rewards/margins": 0.10823094844818115,
+      "rewards/rejected": -1.40617036819458,
+      "step": 2970
+    },
+    {
+      "epoch": 1.5922395049339353,
+      "grad_norm": 8.561026141051741,
+      "learning_rate": 5.334591348440229e-07,
+      "logits/chosen": -0.12392149120569229,
+      "logits/rejected": -0.01384222786873579,
+      "logps/chosen": -1.263944149017334,
+      "logps/rejected": -1.5268821716308594,
+      "loss": 1.5979,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.263944149017334,
+      "rewards/margins": 0.2629380524158478,
+      "rewards/rejected": -1.5268821716308594,
+      "step": 2975
+    },
+    {
+      "epoch": 1.594915537715337,
+      "grad_norm": 10.232686401494488,
+      "learning_rate": 5.319050633623141e-07,
+      "logits/chosen": -0.19973398745059967,
+      "logits/rejected": -0.07975804805755615,
+      "logps/chosen": -1.299760103225708,
+      "logps/rejected": -1.4754284620285034,
+      "loss": 1.6607,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.299760103225708,
+      "rewards/margins": 0.17566823959350586,
+      "rewards/rejected": -1.4754284620285034,
+      "step": 2980
+    },
+    {
+      "epoch": 1.5975915704967387,
+      "grad_norm": 5.869578591150815,
+      "learning_rate": 5.303506823375409e-07,
+      "logits/chosen": -0.20419225096702576,
+      "logits/rejected": -0.06428646296262741,
+      "logps/chosen": -1.3240875005722046,
+      "logps/rejected": -1.422776699066162,
+      "loss": 1.6718,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.3240875005722046,
+      "rewards/margins": 0.0986892357468605,
+      "rewards/rejected": -1.422776699066162,
+      "step": 2985
+    },
+    {
+      "epoch": 1.60026760327814,
+      "grad_norm": 8.404848854739129,
+      "learning_rate": 5.287960068503143e-07,
+      "logits/chosen": -0.1343335062265396,
+      "logits/rejected": 0.011005854234099388,
+      "logps/chosen": -1.2011334896087646,
+      "logps/rejected": -1.452879548072815,
+      "loss": 1.5661,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.2011334896087646,
+      "rewards/margins": 0.25174611806869507,
+      "rewards/rejected": -1.452879548072815,
+      "step": 2990
+    },
+    {
+      "epoch": 1.6029436360595417,
+      "grad_norm": 8.63111795683355,
+      "learning_rate": 5.272410519841032e-07,
+      "logits/chosen": -0.15247437357902527,
+      "logits/rejected": -0.07777571678161621,
+      "logps/chosen": -1.3044708967208862,
+      "logps/rejected": -1.5415220260620117,
+      "loss": 1.6299,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.3044708967208862,
+      "rewards/margins": 0.23705117404460907,
+      "rewards/rejected": -1.5415220260620117,
+      "step": 2995
+    },
+    {
+      "epoch": 1.6056196688409434,
+      "grad_norm": 6.9809615671792775,
+      "learning_rate": 5.256858328250861e-07,
+      "logits/chosen": -0.18980927765369415,
+      "logits/rejected": -0.09071429073810577,
+      "logps/chosen": -1.306738018989563,
+      "logps/rejected": -1.4723536968231201,
+      "loss": 1.6838,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.306738018989563,
+      "rewards/margins": 0.16561564803123474,
+      "rewards/rejected": -1.4723536968231201,
+      "step": 3000
+    },
+    {
+      "epoch": 1.608295701622345,
+      "grad_norm": 7.845303840240266,
+      "learning_rate": 5.241303644620063e-07,
+      "logits/chosen": -0.23705609142780304,
+      "logits/rejected": -0.1214757114648819,
+      "logps/chosen": -1.2059553861618042,
+      "logps/rejected": -1.3896453380584717,
+      "loss": 1.5884,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.2059553861618042,
+      "rewards/margins": 0.18369005620479584,
+      "rewards/rejected": -1.3896453380584717,
+      "step": 3005
+    },
+    {
+      "epoch": 1.6109717344037464,
+      "grad_norm": 8.923428009458794,
+      "learning_rate": 5.225746619860248e-07,
+      "logits/chosen": -0.21868646144866943,
+      "logits/rejected": -0.1172574907541275,
+      "logps/chosen": -1.2465583086013794,
+      "logps/rejected": -1.411179542541504,
+      "loss": 1.636,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.2465583086013794,
+      "rewards/margins": 0.16462130844593048,
+      "rewards/rejected": -1.411179542541504,
+      "step": 3010
+    },
+    {
+      "epoch": 1.6136477671851481,
+      "grad_norm": 8.58111075001241,
+      "learning_rate": 5.210187404905735e-07,
+      "logits/chosen": -0.07596341520547867,
+      "logits/rejected": -0.01020109374076128,
+      "logps/chosen": -1.241079568862915,
+      "logps/rejected": -1.4651001691818237,
+      "loss": 1.5836,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.241079568862915,
+      "rewards/margins": 0.22402064502239227,
+      "rewards/rejected": -1.4651001691818237,
+      "step": 3015
+    },
+    {
+      "epoch": 1.6163237999665496,
+      "grad_norm": 7.440191528787235,
+      "learning_rate": 5.194626150712098e-07,
+      "logits/chosen": -0.21312424540519714,
+      "logits/rejected": -0.08387692272663116,
+      "logps/chosen": -1.241636037826538,
+      "logps/rejected": -1.3631786108016968,
+      "loss": 1.618,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.241636037826538,
+      "rewards/margins": 0.12154249846935272,
+      "rewards/rejected": -1.3631786108016968,
+      "step": 3020
+    },
+    {
+      "epoch": 1.6189998327479511,
+      "grad_norm": 8.20125824115334,
+      "learning_rate": 5.179063008254695e-07,
+      "logits/chosen": -0.15682704746723175,
+      "logits/rejected": -0.04331011325120926,
+      "logps/chosen": -1.2104800939559937,
+      "logps/rejected": -1.360103964805603,
+      "loss": 1.597,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.2104800939559937,
+      "rewards/margins": 0.149623841047287,
+      "rewards/rejected": -1.360103964805603,
+      "step": 3025
+    },
+    {
+      "epoch": 1.6216758655293528,
+      "grad_norm": 5.562431256517953,
+      "learning_rate": 5.163498128527199e-07,
+      "logits/chosen": -0.13923481106758118,
+      "logits/rejected": -0.024134326726198196,
+      "logps/chosen": -1.2932157516479492,
+      "logps/rejected": -1.400172472000122,
+      "loss": 1.6373,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.2932157516479492,
+      "rewards/margins": 0.10695650428533554,
+      "rewards/rejected": -1.400172472000122,
+      "step": 3030
+    },
+    {
+      "epoch": 1.6243518983107543,
+      "grad_norm": 8.08617825073629,
+      "learning_rate": 5.147931662540144e-07,
+      "logits/chosen": -0.03716077283024788,
+      "logits/rejected": 0.04836971312761307,
+      "logps/chosen": -1.2672970294952393,
+      "logps/rejected": -1.349109411239624,
+      "loss": 1.6496,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.2672970294952393,
+      "rewards/margins": 0.08181246370077133,
+      "rewards/rejected": -1.349109411239624,
+      "step": 3035
+    },
+    {
+      "epoch": 1.6270279310921558,
+      "grad_norm": 11.86487644872994,
+      "learning_rate": 5.132363761319449e-07,
+      "logits/chosen": -0.15009820461273193,
+      "logits/rejected": -0.10209157317876816,
+      "logps/chosen": -1.1850813627243042,
+      "logps/rejected": -1.4225887060165405,
+      "loss": 1.5398,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.1850813627243042,
+      "rewards/margins": 0.23750725388526917,
+      "rewards/rejected": -1.4225887060165405,
+      "step": 3040
+    },
+    {
+      "epoch": 1.6297039638735575,
+      "grad_norm": 14.321647563947106,
+      "learning_rate": 5.116794575904962e-07,
+      "logits/chosen": -0.1324605941772461,
+      "logits/rejected": -0.0472918264567852,
+      "logps/chosen": -1.217961072921753,
+      "logps/rejected": -1.329044222831726,
+      "loss": 1.6154,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.217961072921753,
+      "rewards/margins": 0.11108307540416718,
+      "rewards/rejected": -1.329044222831726,
+      "step": 3045
+    },
+    {
+      "epoch": 1.632379996654959,
+      "grad_norm": 9.063482796097663,
+      "learning_rate": 5.101224257348987e-07,
+      "logits/chosen": -0.18629930913448334,
+      "logits/rejected": -0.05383709818124771,
+      "logps/chosen": -1.2813564538955688,
+      "logps/rejected": -1.503534197807312,
+      "loss": 1.6073,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.2813564538955688,
+      "rewards/margins": 0.2221778929233551,
+      "rewards/rejected": -1.503534197807312,
+      "step": 3050
+    },
+    {
+      "epoch": 1.6350560294363605,
+      "grad_norm": 6.12253360363484,
+      "learning_rate": 5.085652956714823e-07,
+      "logits/chosen": -0.19061146676540375,
+      "logits/rejected": -0.07919798791408539,
+      "logps/chosen": -1.217474341392517,
+      "logps/rejected": -1.438800573348999,
+      "loss": 1.5872,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.217474341392517,
+      "rewards/margins": 0.22132618725299835,
+      "rewards/rejected": -1.438800573348999,
+      "step": 3055
+    },
+    {
+      "epoch": 1.6377320622177622,
+      "grad_norm": 6.7458196371965755,
+      "learning_rate": 5.070080825075298e-07,
+      "logits/chosen": -0.19373759627342224,
+      "logits/rejected": -0.05019166320562363,
+      "logps/chosen": -1.2642765045166016,
+      "logps/rejected": -1.4489099979400635,
+      "loss": 1.6179,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.2642765045166016,
+      "rewards/margins": 0.18463341891765594,
+      "rewards/rejected": -1.4489099979400635,
+      "step": 3060
+    },
+    {
+      "epoch": 1.6404080949991637,
+      "grad_norm": 7.8307477062379895,
+      "learning_rate": 5.0545080135113e-07,
+      "logits/chosen": -0.09949559718370438,
+      "logits/rejected": -0.0767357349395752,
+      "logps/chosen": -1.2530882358551025,
+      "logps/rejected": -1.5171091556549072,
+      "loss": 1.5936,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.2530882358551025,
+      "rewards/margins": 0.264021098613739,
+      "rewards/rejected": -1.5171091556549072,
+      "step": 3065
+    },
+    {
+      "epoch": 1.6430841277805652,
+      "grad_norm": 6.924831831509434,
+      "learning_rate": 5.038934673110316e-07,
+      "logits/chosen": -0.2290758341550827,
+      "logits/rejected": -0.13408730924129486,
+      "logps/chosen": -1.249206781387329,
+      "logps/rejected": -1.4324674606323242,
+      "loss": 1.6044,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.249206781387329,
+      "rewards/margins": 0.18326060473918915,
+      "rewards/rejected": -1.4324674606323242,
+      "step": 3070
+    },
+    {
+      "epoch": 1.645760160561967,
+      "grad_norm": 6.550753239242128,
+      "learning_rate": 5.023360954964963e-07,
+      "logits/chosen": -0.25031578540802,
+      "logits/rejected": -0.20224837958812714,
+      "logps/chosen": -1.1969693899154663,
+      "logps/rejected": -1.402588963508606,
+      "loss": 1.5611,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.1969693899154663,
+      "rewards/margins": 0.20561976730823517,
+      "rewards/rejected": -1.402588963508606,
+      "step": 3075
+    },
+    {
+      "epoch": 1.6484361933433684,
+      "grad_norm": 8.22013975849115,
+      "learning_rate": 5.007787010171524e-07,
+      "logits/chosen": -0.2754722237586975,
+      "logits/rejected": -0.11621172726154327,
+      "logps/chosen": -1.1683692932128906,
+      "logps/rejected": -1.361311912536621,
+      "loss": 1.5563,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.1683692932128906,
+      "rewards/margins": 0.19294250011444092,
+      "rewards/rejected": -1.361311912536621,
+      "step": 3080
+    },
+    {
+      "epoch": 1.65111222612477,
+      "grad_norm": 7.9245890872821905,
+      "learning_rate": 4.992212989828477e-07,
+      "logits/chosen": -0.11269180476665497,
+      "logits/rejected": -0.11005918681621552,
+      "logps/chosen": -1.184718370437622,
+      "logps/rejected": -1.3857439756393433,
+      "loss": 1.5516,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.184718370437622,
+      "rewards/margins": 0.20102588832378387,
+      "rewards/rejected": -1.3857439756393433,
+      "step": 3085
+    },
+    {
+      "epoch": 1.6537882589061716,
+      "grad_norm": 7.910985905319788,
+      "learning_rate": 4.976639045035036e-07,
+      "logits/chosen": -0.0895986557006836,
+      "logits/rejected": -0.034634217619895935,
+      "logps/chosen": -1.2269132137298584,
+      "logps/rejected": -1.3635425567626953,
+      "loss": 1.6229,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.2269132137298584,
+      "rewards/margins": 0.1366293728351593,
+      "rewards/rejected": -1.3635425567626953,
+      "step": 3090
+    },
+    {
+      "epoch": 1.6564642916875731,
+      "grad_norm": 8.989701022473724,
+      "learning_rate": 4.961065326889683e-07,
+      "logits/chosen": -0.16246844828128815,
+      "logits/rejected": -0.04997162148356438,
+      "logps/chosen": -1.2454721927642822,
+      "logps/rejected": -1.412967562675476,
+      "loss": 1.6076,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.2454721927642822,
+      "rewards/margins": 0.16749544441699982,
+      "rewards/rejected": -1.412967562675476,
+      "step": 3095
+    },
+    {
+      "epoch": 1.6591403244689746,
+      "grad_norm": 9.286518315162434,
+      "learning_rate": 4.9454919864887e-07,
+      "logits/chosen": -0.3038082718849182,
+      "logits/rejected": -0.1936473548412323,
+      "logps/chosen": -1.2898128032684326,
+      "logps/rejected": -1.420832633972168,
+      "loss": 1.6559,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.2898128032684326,
+      "rewards/margins": 0.1310197114944458,
+      "rewards/rejected": -1.420832633972168,
+      "step": 3100
+    },
+    {
+      "epoch": 1.6618163572503764,
+      "grad_norm": 9.867318872639597,
+      "learning_rate": 4.929919174924701e-07,
+      "logits/chosen": -0.2150929719209671,
+      "logits/rejected": -0.06844820082187653,
+      "logps/chosen": -1.2798480987548828,
+      "logps/rejected": -1.4365907907485962,
+      "loss": 1.6296,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.2798480987548828,
+      "rewards/margins": 0.15674252808094025,
+      "rewards/rejected": -1.4365907907485962,
+      "step": 3105
+    },
+    {
+      "epoch": 1.6644923900317778,
+      "grad_norm": 7.123668267524398,
+      "learning_rate": 4.914347043285177e-07,
+      "logits/chosen": -0.15136371552944183,
+      "logits/rejected": -0.0637313649058342,
+      "logps/chosen": -1.2542929649353027,
+      "logps/rejected": -1.4589463472366333,
+      "loss": 1.6048,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.2542929649353027,
+      "rewards/margins": 0.2046533077955246,
+      "rewards/rejected": -1.4589463472366333,
+      "step": 3110
+    },
+    {
+      "epoch": 1.6671684228131793,
+      "grad_norm": 5.689036503731929,
+      "learning_rate": 4.898775742651013e-07,
+      "logits/chosen": -0.09614956378936768,
+      "logits/rejected": -0.038172896951436996,
+      "logps/chosen": -1.260207176208496,
+      "logps/rejected": -1.4991014003753662,
+      "loss": 1.594,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.260207176208496,
+      "rewards/margins": 0.2388942688703537,
+      "rewards/rejected": -1.4991014003753662,
+      "step": 3115
+    },
+    {
+      "epoch": 1.669844455594581,
+      "grad_norm": 5.621420712895871,
+      "learning_rate": 4.883205424095037e-07,
+      "logits/chosen": -0.22079035639762878,
+      "logits/rejected": -0.10798345506191254,
+      "logps/chosen": -1.3159030675888062,
+      "logps/rejected": -1.482990026473999,
+      "loss": 1.6713,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.3159030675888062,
+      "rewards/margins": 0.16708692908287048,
+      "rewards/rejected": -1.482990026473999,
+      "step": 3120
+    },
+    {
+      "epoch": 1.6725204883759828,
+      "grad_norm": 7.30915810696171,
+      "learning_rate": 4.86763623868055e-07,
+      "logits/chosen": -0.130017951130867,
+      "logits/rejected": -0.050635069608688354,
+      "logps/chosen": -1.3146250247955322,
+      "logps/rejected": -1.5301706790924072,
+      "loss": 1.6536,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.3146250247955322,
+      "rewards/margins": 0.21554553508758545,
+      "rewards/rejected": -1.5301706790924072,
+      "step": 3125
+    },
+    {
+      "epoch": 1.675196521157384,
+      "grad_norm": 7.061782489077253,
+      "learning_rate": 4.852068337459856e-07,
+      "logits/chosen": -0.1131129041314125,
+      "logits/rejected": -0.011948710307478905,
+      "logps/chosen": -1.3172558546066284,
+      "logps/rejected": -1.4849779605865479,
+      "loss": 1.6484,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.3172558546066284,
+      "rewards/margins": 0.16772189736366272,
+      "rewards/rejected": -1.4849779605865479,
+      "step": 3130
+    },
+    {
+      "epoch": 1.6778725539387858,
+      "grad_norm": 7.380763411129059,
+      "learning_rate": 4.8365018714728e-07,
+      "logits/chosen": -0.09311554580926895,
+      "logits/rejected": -0.06127345561981201,
+      "logps/chosen": -1.3208004236221313,
+      "logps/rejected": -1.4602913856506348,
+      "loss": 1.6796,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.3208004236221313,
+      "rewards/margins": 0.13949090242385864,
+      "rewards/rejected": -1.4602913856506348,
+      "step": 3135
+    },
+    {
+      "epoch": 1.6805485867201875,
+      "grad_norm": 5.702850842533426,
+      "learning_rate": 4.820936991745304e-07,
+      "logits/chosen": -0.34844082593917847,
+      "logits/rejected": -0.21953165531158447,
+      "logps/chosen": -1.1999485492706299,
+      "logps/rejected": -1.3034613132476807,
+      "loss": 1.5953,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.1999485492706299,
+      "rewards/margins": 0.10351266711950302,
+      "rewards/rejected": -1.3034613132476807,
+      "step": 3140
+    },
+    {
+      "epoch": 1.6832246195015887,
+      "grad_norm": 8.342386540198653,
+      "learning_rate": 4.8053738492879e-07,
+      "logits/chosen": -0.1601831614971161,
+      "logits/rejected": -0.055739402770996094,
+      "logps/chosen": -1.242384910583496,
+      "logps/rejected": -1.351478934288025,
+      "loss": 1.621,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.242384910583496,
+      "rewards/margins": 0.10909397900104523,
+      "rewards/rejected": -1.351478934288025,
+      "step": 3145
+    },
+    {
+      "epoch": 1.6859006522829905,
+      "grad_norm": 7.9266129658506905,
+      "learning_rate": 4.789812595094265e-07,
+      "logits/chosen": -0.2788141369819641,
+      "logits/rejected": -0.1777099072933197,
+      "logps/chosen": -1.3118698596954346,
+      "logps/rejected": -1.4719301462173462,
+      "loss": 1.6398,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.3118698596954346,
+      "rewards/margins": 0.16006025671958923,
+      "rewards/rejected": -1.4719301462173462,
+      "step": 3150
+    },
+    {
+      "epoch": 1.6885766850643922,
+      "grad_norm": 11.687494404795508,
+      "learning_rate": 4.774253380139752e-07,
+      "logits/chosen": -0.2808185815811157,
+      "logits/rejected": -0.1869959831237793,
+      "logps/chosen": -1.194966435432434,
+      "logps/rejected": -1.4038288593292236,
+      "loss": 1.5647,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.194966435432434,
+      "rewards/margins": 0.20886237919330597,
+      "rewards/rejected": -1.4038288593292236,
+      "step": 3155
+    },
+    {
+      "epoch": 1.6912527178457935,
+      "grad_norm": 8.339394315233186,
+      "learning_rate": 4.758696355379936e-07,
+      "logits/chosen": -0.24925696849822998,
+      "logits/rejected": -0.22909533977508545,
+      "logps/chosen": -1.2405601739883423,
+      "logps/rejected": -1.4713505506515503,
+      "loss": 1.5969,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.2405601739883423,
+      "rewards/margins": 0.2307904064655304,
+      "rewards/rejected": -1.4713505506515503,
+      "step": 3160
+    },
+    {
+      "epoch": 1.6939287506271952,
+      "grad_norm": 6.232115794794186,
+      "learning_rate": 4.743141671749138e-07,
+      "logits/chosen": -0.26409196853637695,
+      "logits/rejected": -0.1972818374633789,
+      "logps/chosen": -1.281771183013916,
+      "logps/rejected": -1.4099771976470947,
+      "loss": 1.6426,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.281771183013916,
+      "rewards/margins": 0.12820599973201752,
+      "rewards/rejected": -1.4099771976470947,
+      "step": 3165
+    },
+    {
+      "epoch": 1.6966047834085969,
+      "grad_norm": 6.145214171888279,
+      "learning_rate": 4.727589480158968e-07,
+      "logits/chosen": -0.22552287578582764,
+      "logits/rejected": -0.1499921828508377,
+      "logps/chosen": -1.2544889450073242,
+      "logps/rejected": -1.4211806058883667,
+      "loss": 1.6312,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.2544889450073242,
+      "rewards/margins": 0.1666916310787201,
+      "rewards/rejected": -1.4211806058883667,
+      "step": 3170
+    },
+    {
+      "epoch": 1.6992808161899984,
+      "grad_norm": 9.181957372766112,
+      "learning_rate": 4.712039931496855e-07,
+      "logits/chosen": -0.2748224139213562,
+      "logits/rejected": -0.20714323222637177,
+      "logps/chosen": -1.2339074611663818,
+      "logps/rejected": -1.3875453472137451,
+      "loss": 1.6198,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.2339074611663818,
+      "rewards/margins": 0.15363790094852448,
+      "rewards/rejected": -1.3875453472137451,
+      "step": 3175
+    },
+    {
+      "epoch": 1.7019568489713999,
+      "grad_norm": 5.559172509042021,
+      "learning_rate": 4.6964931766245905e-07,
+      "logits/chosen": -0.1216423287987709,
+      "logits/rejected": -0.0782778263092041,
+      "logps/chosen": -1.2822221517562866,
+      "logps/rejected": -1.4988685846328735,
+      "loss": 1.6129,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.2822221517562866,
+      "rewards/margins": 0.21664640307426453,
+      "rewards/rejected": -1.4988685846328735,
+      "step": 3180
+    },
+    {
+      "epoch": 1.7046328817528016,
+      "grad_norm": 8.053248508222694,
+      "learning_rate": 4.6809493663768575e-07,
+      "logits/chosen": -0.17452619969844818,
+      "logits/rejected": -0.1666513979434967,
+      "logps/chosen": -1.2001672983169556,
+      "logps/rejected": -1.3536691665649414,
+      "loss": 1.5746,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.2001672983169556,
+      "rewards/margins": 0.15350192785263062,
+      "rewards/rejected": -1.3536691665649414,
+      "step": 3185
+    },
+    {
+      "epoch": 1.707308914534203,
+      "grad_norm": 8.155853541198367,
+      "learning_rate": 4.6654086515597716e-07,
+      "logits/chosen": -0.21809610724449158,
+      "logits/rejected": -0.09687723219394684,
+      "logps/chosen": -1.2203891277313232,
+      "logps/rejected": -1.4710357189178467,
+      "loss": 1.5438,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.2203891277313232,
+      "rewards/margins": 0.2506466507911682,
+      "rewards/rejected": -1.4710357189178467,
+      "step": 3190
+    },
+    {
+      "epoch": 1.7099849473156046,
+      "grad_norm": 5.6339085467216,
+      "learning_rate": 4.6498711829494154e-07,
+      "logits/chosen": -0.23436173796653748,
+      "logits/rejected": -0.14631803333759308,
+      "logps/chosen": -1.227378249168396,
+      "logps/rejected": -1.4630035161972046,
+      "loss": 1.5698,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.227378249168396,
+      "rewards/margins": 0.2356252372264862,
+      "rewards/rejected": -1.4630035161972046,
+      "step": 3195
+    },
+    {
+      "epoch": 1.7126609800970063,
+      "grad_norm": 8.215408437303552,
+      "learning_rate": 4.6343371112903777e-07,
+      "logits/chosen": -0.1575685441493988,
+      "logits/rejected": -0.03138052672147751,
+      "logps/chosen": -1.2763001918792725,
+      "logps/rejected": -1.5757664442062378,
+      "loss": 1.6012,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.2763001918792725,
+      "rewards/margins": 0.2994665205478668,
+      "rewards/rejected": -1.5757664442062378,
+      "step": 3200
+    },
+    {
+      "epoch": 1.7126609800970063,
+      "eval_logits/chosen": 0.11330799758434296,
+      "eval_logits/rejected": 0.18483346700668335,
+      "eval_logps/chosen": -1.3009766340255737,
+      "eval_logps/rejected": -1.4718098640441895,
+      "eval_loss": 1.6493672132492065,
+      "eval_rewards/accuracies": 0.5675074458122253,
+      "eval_rewards/chosen": -1.3009766340255737,
+      "eval_rewards/margins": 0.17083337903022766,
+      "eval_rewards/rejected": -1.4718098640441895,
+      "eval_runtime": 40.5995,
+      "eval_samples_per_second": 33.129,
+      "eval_steps_per_second": 8.301,
+      "step": 3200
+    },
+    {
+      "epoch": 1.7153370128784078,
+      "grad_norm": 5.519402528838374,
+      "learning_rate": 4.618806587294291e-07,
+      "logits/chosen": -0.2710733413696289,
+      "logits/rejected": -0.18044158816337585,
+      "logps/chosen": -1.2949516773223877,
+      "logps/rejected": -1.4818966388702393,
+      "loss": 1.6292,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.2949516773223877,
+      "rewards/margins": 0.1869450807571411,
+      "rewards/rejected": -1.4818966388702393,
+      "step": 3205
+    },
+    {
+      "epoch": 1.7180130456598093,
+      "grad_norm": 9.347170465131176,
+      "learning_rate": 4.603279761638365e-07,
+      "logits/chosen": -0.2634008228778839,
+      "logits/rejected": -0.18680989742279053,
+      "logps/chosen": -1.2629354000091553,
+      "logps/rejected": -1.4384006261825562,
+      "loss": 1.6268,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.2629354000091553,
+      "rewards/margins": 0.17546531558036804,
+      "rewards/rejected": -1.4384006261825562,
+      "step": 3210
+    },
+    {
+      "epoch": 1.720689078441211,
+      "grad_norm": 9.617166522314575,
+      "learning_rate": 4.5877567849639315e-07,
+      "logits/chosen": -0.23709094524383545,
+      "logits/rejected": -0.13341295719146729,
+      "logps/chosen": -1.2301095724105835,
+      "logps/rejected": -1.3809475898742676,
+      "loss": 1.6028,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.2301095724105835,
+      "rewards/margins": 0.15083806216716766,
+      "rewards/rejected": -1.3809475898742676,
+      "step": 3215
+    },
+    {
+      "epoch": 1.7233651112226125,
+      "grad_norm": 6.722448806937953,
+      "learning_rate": 4.572237807874979e-07,
+      "logits/chosen": -0.23734502494335175,
+      "logits/rejected": -0.08293096721172333,
+      "logps/chosen": -1.3151230812072754,
+      "logps/rejected": -1.4774357080459595,
+      "loss": 1.6529,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.3151230812072754,
+      "rewards/margins": 0.16231267154216766,
+      "rewards/rejected": -1.4774357080459595,
+      "step": 3220
+    },
+    {
+      "epoch": 1.726041144004014,
+      "grad_norm": 9.235009754404345,
+      "learning_rate": 4.5567229809366895e-07,
+      "logits/chosen": -0.22738520801067352,
+      "logits/rejected": -0.12039558589458466,
+      "logps/chosen": -1.1815749406814575,
+      "logps/rejected": -1.3902910947799683,
+      "loss": 1.5459,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.1815749406814575,
+      "rewards/margins": 0.2087162286043167,
+      "rewards/rejected": -1.3902910947799683,
+      "step": 3225
+    },
+    {
+      "epoch": 1.7287171767854157,
+      "grad_norm": 6.67073380977587,
+      "learning_rate": 4.541212454673984e-07,
+      "logits/chosen": -0.23986825346946716,
+      "logits/rejected": -0.1233188733458519,
+      "logps/chosen": -1.22847580909729,
+      "logps/rejected": -1.4644906520843506,
+      "loss": 1.5682,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.22847580909729,
+      "rewards/margins": 0.23601479828357697,
+      "rewards/rejected": -1.4644906520843506,
+      "step": 3230
+    },
+    {
+      "epoch": 1.7313932095668172,
+      "grad_norm": 8.575828281719883,
+      "learning_rate": 4.525706379570055e-07,
+      "logits/chosen": -0.1955959051847458,
+      "logits/rejected": -0.15789765119552612,
+      "logps/chosen": -1.2535419464111328,
+      "logps/rejected": -1.437584638595581,
+      "loss": 1.6022,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.2535419464111328,
+      "rewards/margins": 0.18404273688793182,
+      "rewards/rejected": -1.437584638595581,
+      "step": 3235
+    },
+    {
+      "epoch": 1.7340692423482187,
+      "grad_norm": 6.262181768654312,
+      "learning_rate": 4.510204906064911e-07,
+      "logits/chosen": -0.11629863828420639,
+      "logits/rejected": -0.04444526880979538,
+      "logps/chosen": -1.1999154090881348,
+      "logps/rejected": -1.4560317993164062,
+      "loss": 1.5254,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.1999154090881348,
+      "rewards/margins": 0.25611644983291626,
+      "rewards/rejected": -1.4560317993164062,
+      "step": 3240
+    },
+    {
+      "epoch": 1.7367452751296204,
+      "grad_norm": 9.168006641027242,
+      "learning_rate": 4.4947081845539177e-07,
+      "logits/chosen": -0.30351483821868896,
+      "logits/rejected": -0.1936475783586502,
+      "logps/chosen": -1.205866813659668,
+      "logps/rejected": -1.3990415334701538,
+      "loss": 1.5854,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.205866813659668,
+      "rewards/margins": 0.193174809217453,
+      "rewards/rejected": -1.3990415334701538,
+      "step": 3245
+    },
+    {
+      "epoch": 1.739421307911022,
+      "grad_norm": 7.1099536920776725,
+      "learning_rate": 4.479216365386333e-07,
+      "logits/chosen": -0.10585144907236099,
+      "logits/rejected": -0.015230861492455006,
+      "logps/chosen": -1.2517902851104736,
+      "logps/rejected": -1.476215124130249,
+      "loss": 1.5946,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.2517902851104736,
+      "rewards/margins": 0.2244250327348709,
+      "rewards/rejected": -1.476215124130249,
+      "step": 3250
+    },
+    {
+      "epoch": 1.7420973406924234,
+      "grad_norm": 7.28474636819934,
+      "learning_rate": 4.4637295988638555e-07,
+      "logits/chosen": -0.12763206660747528,
+      "logits/rejected": -0.06798548251390457,
+      "logps/chosen": -1.3406620025634766,
+      "logps/rejected": -1.4056285619735718,
+      "loss": 1.6821,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.3406620025634766,
+      "rewards/margins": 0.0649665966629982,
+      "rewards/rejected": -1.4056285619735718,
+      "step": 3255
+    },
+    {
+      "epoch": 1.744773373473825,
+      "grad_norm": 8.174404159145823,
+      "learning_rate": 4.4482480352391623e-07,
+      "logits/chosen": -0.2601187229156494,
+      "logits/rejected": -0.1402393877506256,
+      "logps/chosen": -1.2974830865859985,
+      "logps/rejected": -1.3821454048156738,
+      "loss": 1.642,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.2974830865859985,
+      "rewards/margins": 0.0846622958779335,
+      "rewards/rejected": -1.3821454048156738,
+      "step": 3260
+    },
+    {
+      "epoch": 1.7474494062552266,
+      "grad_norm": 11.452650486684668,
+      "learning_rate": 4.4327718247144507e-07,
+      "logits/chosen": -0.14017020165920258,
+      "logits/rejected": -0.049223341047763824,
+      "logps/chosen": -1.1975616216659546,
+      "logps/rejected": -1.4038350582122803,
+      "loss": 1.5632,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.1975616216659546,
+      "rewards/margins": 0.20627336204051971,
+      "rewards/rejected": -1.4038350582122803,
+      "step": 3265
+    },
+    {
+      "epoch": 1.750125439036628,
+      "grad_norm": 8.279683070973517,
+      "learning_rate": 4.417301117439984e-07,
+      "logits/chosen": -0.15070870518684387,
+      "logits/rejected": -0.019910674542188644,
+      "logps/chosen": -1.157780647277832,
+      "logps/rejected": -1.3996648788452148,
+      "loss": 1.5372,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.157780647277832,
+      "rewards/margins": 0.2418842613697052,
+      "rewards/rejected": -1.3996648788452148,
+      "step": 3270
+    },
+    {
+      "epoch": 1.7528014718180298,
+      "grad_norm": 9.072661083572072,
+      "learning_rate": 4.401836063512631e-07,
+      "logits/chosen": -0.17229340970516205,
+      "logits/rejected": 0.08592073619365692,
+      "logps/chosen": -1.2706421613693237,
+      "logps/rejected": -1.3676345348358154,
+      "loss": 1.6445,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.2706421613693237,
+      "rewards/margins": 0.09699241071939468,
+      "rewards/rejected": -1.3676345348358154,
+      "step": 3275
+    },
+    {
+      "epoch": 1.7554775045994313,
+      "grad_norm": 8.30874909165034,
+      "learning_rate": 4.386376812974413e-07,
+      "logits/chosen": -0.16627468168735504,
+      "logits/rejected": -0.08861502259969711,
+      "logps/chosen": -1.2122783660888672,
+      "logps/rejected": -1.3982435464859009,
+      "loss": 1.5866,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.2122783660888672,
+      "rewards/margins": 0.18596510589122772,
+      "rewards/rejected": -1.3982435464859009,
+      "step": 3280
+    },
+    {
+      "epoch": 1.7581535373808328,
+      "grad_norm": 7.630394959614666,
+      "learning_rate": 4.370923515811048e-07,
+      "logits/chosen": -0.19240662455558777,
+      "logits/rejected": -0.01533288974314928,
+      "logps/chosen": -1.2119776010513306,
+      "logps/rejected": -1.4352428913116455,
+      "loss": 1.6006,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.2119776010513306,
+      "rewards/margins": 0.22326524555683136,
+      "rewards/rejected": -1.4352428913116455,
+      "step": 3285
+    },
+    {
+      "epoch": 1.7608295701622345,
+      "grad_norm": 8.218490443754595,
+      "learning_rate": 4.35547632195049e-07,
+      "logits/chosen": -0.1405266672372818,
+      "logits/rejected": -0.04814616218209267,
+      "logps/chosen": -1.2359240055084229,
+      "logps/rejected": -1.331750512123108,
+      "loss": 1.6362,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.2359240055084229,
+      "rewards/margins": 0.0958266407251358,
+      "rewards/rejected": -1.331750512123108,
+      "step": 3290
+    },
+    {
+      "epoch": 1.763505602943636,
+      "grad_norm": 9.22563905261601,
+      "learning_rate": 4.340035381261484e-07,
+      "logits/chosen": -0.1587071567773819,
+      "logits/rejected": -0.11247760057449341,
+      "logps/chosen": -1.3131606578826904,
+      "logps/rejected": -1.4168140888214111,
+      "loss": 1.7043,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.3131606578826904,
+      "rewards/margins": 0.10365326702594757,
+      "rewards/rejected": -1.4168140888214111,
+      "step": 3295
+    },
+    {
+      "epoch": 1.7661816357250375,
+      "grad_norm": 8.056133465059338,
+      "learning_rate": 4.324600843552104e-07,
+      "logits/chosen": -0.24770304560661316,
+      "logits/rejected": -0.13748589158058167,
+      "logps/chosen": -1.3128284215927124,
+      "logps/rejected": -1.464592695236206,
+      "loss": 1.6644,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.3128284215927124,
+      "rewards/margins": 0.15176445245742798,
+      "rewards/rejected": -1.464592695236206,
+      "step": 3300
+    },
+    {
+      "epoch": 1.7688576685064392,
+      "grad_norm": 9.249667644920944,
+      "learning_rate": 4.309172858568302e-07,
+      "logits/chosen": -0.23670992255210876,
+      "logits/rejected": -0.12905147671699524,
+      "logps/chosen": -1.2816505432128906,
+      "logps/rejected": -1.3982784748077393,
+      "loss": 1.6566,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.2816505432128906,
+      "rewards/margins": 0.11662785708904266,
+      "rewards/rejected": -1.3982784748077393,
+      "step": 3305
+    },
+    {
+      "epoch": 1.771533701287841,
+      "grad_norm": 7.673087844994584,
+      "learning_rate": 4.293751575992455e-07,
+      "logits/chosen": -0.09383638203144073,
+      "logits/rejected": -0.05546841025352478,
+      "logps/chosen": -1.2519562244415283,
+      "logps/rejected": -1.4098784923553467,
+      "loss": 1.6148,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.2519562244415283,
+      "rewards/margins": 0.15792234241962433,
+      "rewards/rejected": -1.4098784923553467,
+      "step": 3310
+    },
+    {
+      "epoch": 1.7742097340692422,
+      "grad_norm": 9.027262073359903,
+      "learning_rate": 4.278337145441916e-07,
+      "logits/chosen": -0.2571185231208801,
+      "logits/rejected": -0.142175555229187,
+      "logps/chosen": -1.216233253479004,
+      "logps/rejected": -1.3984395265579224,
+      "loss": 1.5919,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.216233253479004,
+      "rewards/margins": 0.18220646679401398,
+      "rewards/rejected": -1.3984395265579224,
+      "step": 3315
+    },
+    {
+      "epoch": 1.776885766850644,
+      "grad_norm": 7.06508547800236,
+      "learning_rate": 4.262929716467556e-07,
+      "logits/chosen": -0.1786039024591446,
+      "logits/rejected": -0.02842709794640541,
+      "logps/chosen": -1.2456203699111938,
+      "logps/rejected": -1.4679776430130005,
+      "loss": 1.5921,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.2456203699111938,
+      "rewards/margins": 0.22235722839832306,
+      "rewards/rejected": -1.4679776430130005,
+      "step": 3320
+    },
+    {
+      "epoch": 1.7795617996320456,
+      "grad_norm": 8.012772750013468,
+      "learning_rate": 4.247529438552321e-07,
+      "logits/chosen": -0.26813259720802307,
+      "logits/rejected": -0.11593590676784515,
+      "logps/chosen": -1.2606017589569092,
+      "logps/rejected": -1.45313560962677,
+      "loss": 1.6097,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.2606017589569092,
+      "rewards/margins": 0.1925339698791504,
+      "rewards/rejected": -1.45313560962677,
+      "step": 3325
+    },
+    {
+      "epoch": 1.782237832413447,
+      "grad_norm": 9.729844540213076,
+      "learning_rate": 4.232136461109773e-07,
+      "logits/chosen": -0.13743022084236145,
+      "logits/rejected": -0.04654218629002571,
+      "logps/chosen": -1.1681630611419678,
+      "logps/rejected": -1.412596583366394,
+      "loss": 1.5242,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.1681630611419678,
+      "rewards/margins": 0.2444334775209427,
+      "rewards/rejected": -1.412596583366394,
+      "step": 3330
+    },
+    {
+      "epoch": 1.7849138651948486,
+      "grad_norm": 11.710305594086776,
+      "learning_rate": 4.216750933482646e-07,
+      "logits/chosen": -0.1769091784954071,
+      "logits/rejected": -0.03825043514370918,
+      "logps/chosen": -1.2831181287765503,
+      "logps/rejected": -1.4444267749786377,
+      "loss": 1.6162,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.2831181287765503,
+      "rewards/margins": 0.16130872070789337,
+      "rewards/rejected": -1.4444267749786377,
+      "step": 3335
+    },
+    {
+      "epoch": 1.7875898979762503,
+      "grad_norm": 6.253784938341751,
+      "learning_rate": 4.2013730049413986e-07,
+      "logits/chosen": -0.12877388298511505,
+      "logits/rejected": -0.01266949437558651,
+      "logps/chosen": -1.2255029678344727,
+      "logps/rejected": -1.4497065544128418,
+      "loss": 1.5884,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.2255029678344727,
+      "rewards/margins": 0.22420351207256317,
+      "rewards/rejected": -1.4497065544128418,
+      "step": 3340
+    },
+    {
+      "epoch": 1.7902659307576518,
+      "grad_norm": 8.849273462673999,
+      "learning_rate": 4.1860028246827594e-07,
+      "logits/chosen": -0.1709320843219757,
+      "logits/rejected": -0.03634721040725708,
+      "logps/chosen": -1.1567004919052124,
+      "logps/rejected": -1.364342451095581,
+      "loss": 1.5424,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.1567004919052124,
+      "rewards/margins": 0.20764203369617462,
+      "rewards/rejected": -1.364342451095581,
+      "step": 3345
+    },
+    {
+      "epoch": 1.7929419635390533,
+      "grad_norm": 8.5624059736707,
+      "learning_rate": 4.170640541828285e-07,
+      "logits/chosen": -0.27934056520462036,
+      "logits/rejected": -0.15984955430030823,
+      "logps/chosen": -1.321946620941162,
+      "logps/rejected": -1.4497636556625366,
+      "loss": 1.6759,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.321946620941162,
+      "rewards/margins": 0.1278170347213745,
+      "rewards/rejected": -1.4497636556625366,
+      "step": 3350
+    },
+    {
+      "epoch": 1.795617996320455,
+      "grad_norm": 10.439526585881291,
+      "learning_rate": 4.1552863054229116e-07,
+      "logits/chosen": -0.035465117543935776,
+      "logits/rejected": -0.004923814907670021,
+      "logps/chosen": -1.3120362758636475,
+      "logps/rejected": -1.3973262310028076,
+      "loss": 1.7083,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -1.3120362758636475,
+      "rewards/margins": 0.08529011905193329,
+      "rewards/rejected": -1.3973262310028076,
+      "step": 3355
+    },
+    {
+      "epoch": 1.7982940291018565,
+      "grad_norm": 8.191935181400508,
+      "learning_rate": 4.139940264433508e-07,
+      "logits/chosen": -0.17950890958309174,
+      "logits/rejected": -0.0031684041023254395,
+      "logps/chosen": -1.2063032388687134,
+      "logps/rejected": -1.3835880756378174,
+      "loss": 1.5792,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.2063032388687134,
+      "rewards/margins": 0.17728491127490997,
+      "rewards/rejected": -1.3835880756378174,
+      "step": 3360
+    },
+    {
+      "epoch": 1.800970061883258,
+      "grad_norm": 5.825131917702379,
+      "learning_rate": 4.1246025677474303e-07,
+      "logits/chosen": -0.19442106783390045,
+      "logits/rejected": -0.06658891588449478,
+      "logps/chosen": -1.2423062324523926,
+      "logps/rejected": -1.4395949840545654,
+      "loss": 1.5854,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.2423062324523926,
+      "rewards/margins": 0.1972886472940445,
+      "rewards/rejected": -1.4395949840545654,
+      "step": 3365
+    },
+    {
+      "epoch": 1.8036460946646597,
+      "grad_norm": 6.410860625207539,
+      "learning_rate": 4.10927336417108e-07,
+      "logits/chosen": -0.17737777531147003,
+      "logits/rejected": -0.05374649167060852,
+      "logps/chosen": -1.2333533763885498,
+      "logps/rejected": -1.3829739093780518,
+      "loss": 1.6015,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.2333533763885498,
+      "rewards/margins": 0.14962056279182434,
+      "rewards/rejected": -1.3829739093780518,
+      "step": 3370
+    },
+    {
+      "epoch": 1.8063221274460612,
+      "grad_norm": 7.866950645857972,
+      "learning_rate": 4.093952802428457e-07,
+      "logits/chosen": -0.010890421457588673,
+      "logits/rejected": 0.023261893540620804,
+      "logps/chosen": -1.268868088722229,
+      "logps/rejected": -1.3623992204666138,
+      "loss": 1.6544,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.268868088722229,
+      "rewards/margins": 0.09353115409612656,
+      "rewards/rejected": -1.3623992204666138,
+      "step": 3375
+    },
+    {
+      "epoch": 1.8089981602274627,
+      "grad_norm": 5.672252950162126,
+      "learning_rate": 4.0786410311597184e-07,
+      "logits/chosen": -0.24759213626384735,
+      "logits/rejected": -0.1401231288909912,
+      "logps/chosen": -1.2340792417526245,
+      "logps/rejected": -1.4476680755615234,
+      "loss": 1.5787,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.2340792417526245,
+      "rewards/margins": 0.21358875930309296,
+      "rewards/rejected": -1.4476680755615234,
+      "step": 3380
+    },
+    {
+      "epoch": 1.8116741930088645,
+      "grad_norm": 7.211360904627205,
+      "learning_rate": 4.063338198919737e-07,
+      "logits/chosen": -0.2078475058078766,
+      "logits/rejected": -0.1844664216041565,
+      "logps/chosen": -1.294800043106079,
+      "logps/rejected": -1.4443811178207397,
+      "loss": 1.6502,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.294800043106079,
+      "rewards/margins": 0.14958105981349945,
+      "rewards/rejected": -1.4443811178207397,
+      "step": 3385
+    },
+    {
+      "epoch": 1.814350225790266,
+      "grad_norm": 10.003239488954332,
+      "learning_rate": 4.0480444541766575e-07,
+      "logits/chosen": -0.15505781769752502,
+      "logits/rejected": -0.05917396396398544,
+      "logps/chosen": -1.3137118816375732,
+      "logps/rejected": -1.4140517711639404,
+      "loss": 1.6835,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.3137118816375732,
+      "rewards/margins": 0.10033991187810898,
+      "rewards/rejected": -1.4140517711639404,
+      "step": 3390
+    },
+    {
+      "epoch": 1.8170262585716674,
+      "grad_norm": 8.940401448632166,
+      "learning_rate": 4.0327599453104606e-07,
+      "logits/chosen": -0.2057075947523117,
+      "logits/rejected": -0.13585631549358368,
+      "logps/chosen": -1.1912782192230225,
+      "logps/rejected": -1.4137266874313354,
+      "loss": 1.5635,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.1912782192230225,
+      "rewards/margins": 0.22244830429553986,
+      "rewards/rejected": -1.4137266874313354,
+      "step": 3395
+    },
+    {
+      "epoch": 1.8197022913530692,
+      "grad_norm": 8.003560686277208,
+      "learning_rate": 4.017484820611514e-07,
+      "logits/chosen": -0.1746649146080017,
+      "logits/rejected": -0.07764903455972672,
+      "logps/chosen": -1.2590306997299194,
+      "logps/rejected": -1.4080326557159424,
+      "loss": 1.6141,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.2590306997299194,
+      "rewards/margins": 0.14900176227092743,
+      "rewards/rejected": -1.4080326557159424,
+      "step": 3400
+    },
+    {
+      "epoch": 1.8223783241344707,
+      "grad_norm": 11.346465262398201,
+      "learning_rate": 4.002219228279148e-07,
+      "logits/chosen": -0.18884289264678955,
+      "logits/rejected": -0.061424195766448975,
+      "logps/chosen": -1.2502999305725098,
+      "logps/rejected": -1.4279358386993408,
+      "loss": 1.609,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.2502999305725098,
+      "rewards/margins": 0.17763593792915344,
+      "rewards/rejected": -1.4279358386993408,
+      "step": 3405
+    },
+    {
+      "epoch": 1.8250543569158721,
+      "grad_norm": 8.144778692952709,
+      "learning_rate": 3.9869633164202045e-07,
+      "logits/chosen": -0.18082715570926666,
+      "logits/rejected": -0.0020881250966340303,
+      "logps/chosen": -1.3831933736801147,
+      "logps/rejected": -1.4894465208053589,
+      "loss": 1.7307,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.3831933736801147,
+      "rewards/margins": 0.10625322163105011,
+      "rewards/rejected": -1.4894465208053589,
+      "step": 3410
+    },
+    {
+      "epoch": 1.8277303896972739,
+      "grad_norm": 7.661754333515176,
+      "learning_rate": 3.9717172330476077e-07,
+      "logits/chosen": -0.16274294257164001,
+      "logits/rejected": -0.07826801389455795,
+      "logps/chosen": -1.2358801364898682,
+      "logps/rejected": -1.4467062950134277,
+      "loss": 1.5922,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.2358801364898682,
+      "rewards/margins": 0.2108263075351715,
+      "rewards/rejected": -1.4467062950134277,
+      "step": 3415
+    },
+    {
+      "epoch": 1.8304064224786754,
+      "grad_norm": 9.58276576176987,
+      "learning_rate": 3.956481126078927e-07,
+      "logits/chosen": -0.11591336876153946,
+      "logits/rejected": -0.014336993917822838,
+      "logps/chosen": -1.2792669534683228,
+      "logps/rejected": -1.5205464363098145,
+      "loss": 1.6288,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.2792669534683228,
+      "rewards/margins": 0.24127936363220215,
+      "rewards/rejected": -1.5205464363098145,
+      "step": 3420
+    },
+    {
+      "epoch": 1.8330824552600768,
+      "grad_norm": 6.151300442369192,
+      "learning_rate": 3.941255143334937e-07,
+      "logits/chosen": -0.22207434475421906,
+      "logits/rejected": -0.18767404556274414,
+      "logps/chosen": -1.242661714553833,
+      "logps/rejected": -1.417602777481079,
+      "loss": 1.6019,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.242661714553833,
+      "rewards/margins": 0.17494121193885803,
+      "rewards/rejected": -1.417602777481079,
+      "step": 3425
+    },
+    {
+      "epoch": 1.8357584880414786,
+      "grad_norm": 10.485462818829816,
+      "learning_rate": 3.9260394325381895e-07,
+      "logits/chosen": -0.17935669422149658,
+      "logits/rejected": -0.07608787715435028,
+      "logps/chosen": -1.3051313161849976,
+      "logps/rejected": -1.4965819120407104,
+      "loss": 1.637,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.3051313161849976,
+      "rewards/margins": 0.19145064055919647,
+      "rewards/rejected": -1.4965819120407104,
+      "step": 3430
+    },
+    {
+      "epoch": 1.83843452082288,
+      "grad_norm": 10.217071873269104,
+      "learning_rate": 3.9108341413115784e-07,
+      "logits/chosen": -0.19460782408714294,
+      "logits/rejected": -0.12509635090827942,
+      "logps/chosen": -1.2595460414886475,
+      "logps/rejected": -1.4745066165924072,
+      "loss": 1.5893,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.2595460414886475,
+      "rewards/margins": 0.21496033668518066,
+      "rewards/rejected": -1.4745066165924072,
+      "step": 3435
+    },
+    {
+      "epoch": 1.8411105536042816,
+      "grad_norm": 10.063470604313583,
+      "learning_rate": 3.895639417176905e-07,
+      "logits/chosen": -0.2412361204624176,
+      "logits/rejected": -0.18948474526405334,
+      "logps/chosen": -1.1810823678970337,
+      "logps/rejected": -1.4140219688415527,
+      "loss": 1.5661,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.1810823678970337,
+      "rewards/margins": 0.2329397201538086,
+      "rewards/rejected": -1.4140219688415527,
+      "step": 3440
+    },
+    {
+      "epoch": 1.8437865863856833,
+      "grad_norm": 8.01386847459897,
+      "learning_rate": 3.8804554075534497e-07,
+      "logits/chosen": -0.23835010826587677,
+      "logits/rejected": -0.043347861617803574,
+      "logps/chosen": -1.2297178506851196,
+      "logps/rejected": -1.407577395439148,
+      "loss": 1.5735,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.2297178506851196,
+      "rewards/margins": 0.17785969376564026,
+      "rewards/rejected": -1.407577395439148,
+      "step": 3445
+    },
+    {
+      "epoch": 1.8464626191670848,
+      "grad_norm": 10.227781776336018,
+      "learning_rate": 3.8652822597565403e-07,
+      "logits/chosen": -0.30860528349876404,
+      "logits/rejected": -0.15392982959747314,
+      "logps/chosen": -1.250962257385254,
+      "logps/rejected": -1.5010929107666016,
+      "loss": 1.585,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.250962257385254,
+      "rewards/margins": 0.25013068318367004,
+      "rewards/rejected": -1.5010929107666016,
+      "step": 3450
+    },
+    {
+      "epoch": 1.8491386519484863,
+      "grad_norm": 6.974518358323705,
+      "learning_rate": 3.850120120996123e-07,
+      "logits/chosen": -0.18968996405601501,
+      "logits/rejected": -0.06573452800512314,
+      "logps/chosen": -1.4123960733413696,
+      "logps/rejected": -1.6055738925933838,
+      "loss": 1.7374,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.4123960733413696,
+      "rewards/margins": 0.19317783415317535,
+      "rewards/rejected": -1.6055738925933838,
+      "step": 3455
+    },
+    {
+      "epoch": 1.851814684729888,
+      "grad_norm": 10.776685643361438,
+      "learning_rate": 3.8349691383753356e-07,
+      "logits/chosen": -0.06698723137378693,
+      "logits/rejected": 0.0407029464840889,
+      "logps/chosen": -1.2353590726852417,
+      "logps/rejected": -1.4258449077606201,
+      "loss": 1.6391,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.2353590726852417,
+      "rewards/margins": 0.19048604369163513,
+      "rewards/rejected": -1.4258449077606201,
+      "step": 3460
+    },
+    {
+      "epoch": 1.8544907175112895,
+      "grad_norm": 6.886761549669779,
+      "learning_rate": 3.819829458889078e-07,
+      "logits/chosen": -0.2288968861103058,
+      "logits/rejected": -0.11740916967391968,
+      "logps/chosen": -1.2035908699035645,
+      "logps/rejected": -1.3677371740341187,
+      "loss": 1.5767,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.2035908699035645,
+      "rewards/margins": 0.16414625942707062,
+      "rewards/rejected": -1.3677371740341187,
+      "step": 3465
+    },
+    {
+      "epoch": 1.857166750292691,
+      "grad_norm": 7.653110287741205,
+      "learning_rate": 3.804701229422585e-07,
+      "logits/chosen": -0.24052855372428894,
+      "logits/rejected": -0.15510748326778412,
+      "logps/chosen": -1.3313173055648804,
+      "logps/rejected": -1.4727556705474854,
+      "loss": 1.6644,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.3313173055648804,
+      "rewards/margins": 0.1414383500814438,
+      "rewards/rejected": -1.4727556705474854,
+      "step": 3470
+    },
+    {
+      "epoch": 1.8598427830740927,
+      "grad_norm": 6.938993849604643,
+      "learning_rate": 3.789584596750007e-07,
+      "logits/chosen": -0.2517121434211731,
+      "logits/rejected": -0.1962416172027588,
+      "logps/chosen": -1.2569118738174438,
+      "logps/rejected": -1.4523240327835083,
+      "loss": 1.6172,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.2569118738174438,
+      "rewards/margins": 0.19541211426258087,
+      "rewards/rejected": -1.4523240327835083,
+      "step": 3475
+    },
+    {
+      "epoch": 1.8625188158554944,
+      "grad_norm": 7.922434750323968,
+      "learning_rate": 3.77447970753298e-07,
+      "logits/chosen": -0.1458994597196579,
+      "logits/rejected": -0.12055318057537079,
+      "logps/chosen": -1.2811120748519897,
+      "logps/rejected": -1.4992064237594604,
+      "loss": 1.6195,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.2811120748519897,
+      "rewards/margins": 0.2180943489074707,
+      "rewards/rejected": -1.4992064237594604,
+      "step": 3480
+    },
+    {
+      "epoch": 1.8651948486368957,
+      "grad_norm": 10.217164584913801,
+      "learning_rate": 3.7593867083192057e-07,
+      "logits/chosen": -0.16993644833564758,
+      "logits/rejected": -0.08232876658439636,
+      "logps/chosen": -1.2428160905838013,
+      "logps/rejected": -1.4211010932922363,
+      "loss": 1.6092,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.2428160905838013,
+      "rewards/margins": 0.17828497290611267,
+      "rewards/rejected": -1.4211010932922363,
+      "step": 3485
+    },
+    {
+      "epoch": 1.8678708814182974,
+      "grad_norm": 6.790309217738528,
+      "learning_rate": 3.7443057455410276e-07,
+      "logits/chosen": -0.15006773173809052,
+      "logits/rejected": -0.043843645602464676,
+      "logps/chosen": -1.300969123840332,
+      "logps/rejected": -1.3989123106002808,
+      "loss": 1.6606,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.300969123840332,
+      "rewards/margins": 0.09794305264949799,
+      "rewards/rejected": -1.3989123106002808,
+      "step": 3490
+    },
+    {
+      "epoch": 1.870546914199699,
+      "grad_norm": 8.197605176941899,
+      "learning_rate": 3.7292369655140145e-07,
+      "logits/chosen": -0.2248447686433792,
+      "logits/rejected": -0.07948695868253708,
+      "logps/chosen": -1.252953290939331,
+      "logps/rejected": -1.4324767589569092,
+      "loss": 1.5976,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.252953290939331,
+      "rewards/margins": 0.1795234978199005,
+      "rewards/rejected": -1.4324767589569092,
+      "step": 3495
+    },
+    {
+      "epoch": 1.8732229469811004,
+      "grad_norm": 8.118952561685207,
+      "learning_rate": 3.714180514435534e-07,
+      "logits/chosen": -0.15942063927650452,
+      "logits/rejected": -0.023639973253011703,
+      "logps/chosen": -1.298370599746704,
+      "logps/rejected": -1.5295395851135254,
+      "loss": 1.6139,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.298370599746704,
+      "rewards/margins": 0.2311691790819168,
+      "rewards/rejected": -1.5295395851135254,
+      "step": 3500
+    },
+    {
+      "epoch": 1.875898979762502,
+      "grad_norm": 9.018634962880915,
+      "learning_rate": 3.6991365383833426e-07,
+      "logits/chosen": -0.15019235014915466,
+      "logits/rejected": -0.05135111138224602,
+      "logps/chosen": -1.2949392795562744,
+      "logps/rejected": -1.4940239191055298,
+      "loss": 1.6456,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.2949392795562744,
+      "rewards/margins": 0.19908465445041656,
+      "rewards/rejected": -1.4940239191055298,
+      "step": 3505
+    },
+    {
+      "epoch": 1.8785750125439038,
+      "grad_norm": 9.397012119740829,
+      "learning_rate": 3.684105183314162e-07,
+      "logits/chosen": -0.19316525757312775,
+      "logits/rejected": -0.1316845864057541,
+      "logps/chosen": -1.2419188022613525,
+      "logps/rejected": -1.3934087753295898,
+      "loss": 1.5919,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.2419188022613525,
+      "rewards/margins": 0.15148988366127014,
+      "rewards/rejected": -1.3934087753295898,
+      "step": 3510
+    },
+    {
+      "epoch": 1.881251045325305,
+      "grad_norm": 9.365950918947213,
+      "learning_rate": 3.669086595062263e-07,
+      "logits/chosen": -0.16677063703536987,
+      "logits/rejected": 0.004392133094370365,
+      "logps/chosen": -1.2826802730560303,
+      "logps/rejected": -1.4858967065811157,
+      "loss": 1.6239,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.2826802730560303,
+      "rewards/margins": 0.20321647822856903,
+      "rewards/rejected": -1.4858967065811157,
+      "step": 3515
+    },
+    {
+      "epoch": 1.8839270781067068,
+      "grad_norm": 8.981100395468093,
+      "learning_rate": 3.654080919338056e-07,
+      "logits/chosen": -0.2394132912158966,
+      "logits/rejected": -0.13284596800804138,
+      "logps/chosen": -1.2914131879806519,
+      "logps/rejected": -1.4775323867797852,
+      "loss": 1.6358,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.2914131879806519,
+      "rewards/margins": 0.18611934781074524,
+      "rewards/rejected": -1.4775323867797852,
+      "step": 3520
+    },
+    {
+      "epoch": 1.8866031108881085,
+      "grad_norm": 7.288596464022811,
+      "learning_rate": 3.639088301726673e-07,
+      "logits/chosen": -0.1558699905872345,
+      "logits/rejected": 0.013392647728323936,
+      "logps/chosen": -1.255486249923706,
+      "logps/rejected": -1.4498775005340576,
+      "loss": 1.6129,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.255486249923706,
+      "rewards/margins": 0.19439134001731873,
+      "rewards/rejected": -1.4498775005340576,
+      "step": 3525
+    },
+    {
+      "epoch": 1.88927914366951,
+      "grad_norm": 11.646149377669992,
+      "learning_rate": 3.624108887686556e-07,
+      "logits/chosen": -0.14714086055755615,
+      "logits/rejected": -0.0844908356666565,
+      "logps/chosen": -1.2383390665054321,
+      "logps/rejected": -1.4483082294464111,
+      "loss": 1.5954,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.2383390665054321,
+      "rewards/margins": 0.2099691927433014,
+      "rewards/rejected": -1.4483082294464111,
+      "step": 3530
+    },
+    {
+      "epoch": 1.8919551764509115,
+      "grad_norm": 6.508255128650294,
+      "learning_rate": 3.6091428225480433e-07,
+      "logits/chosen": -0.23644837737083435,
+      "logits/rejected": -0.12065120041370392,
+      "logps/chosen": -1.2052524089813232,
+      "logps/rejected": -1.347082257270813,
+      "loss": 1.6029,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.2052524089813232,
+      "rewards/margins": 0.14182987809181213,
+      "rewards/rejected": -1.347082257270813,
+      "step": 3535
+    },
+    {
+      "epoch": 1.8946312092323132,
+      "grad_norm": 10.987640646933064,
+      "learning_rate": 3.5941902515119674e-07,
+      "logits/chosen": -0.17603799700737,
+      "logits/rejected": 0.02899671159684658,
+      "logps/chosen": -1.2338905334472656,
+      "logps/rejected": -1.4330108165740967,
+      "loss": 1.5923,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.2338905334472656,
+      "rewards/margins": 0.19912008941173553,
+      "rewards/rejected": -1.4330108165740967,
+      "step": 3540
+    },
+    {
+      "epoch": 1.8973072420137147,
+      "grad_norm": 9.210600086405625,
+      "learning_rate": 3.5792513196482373e-07,
+      "logits/chosen": -0.3058917820453644,
+      "logits/rejected": -0.07458896934986115,
+      "logps/chosen": -1.236205816268921,
+      "logps/rejected": -1.3533177375793457,
+      "loss": 1.6074,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.236205816268921,
+      "rewards/margins": 0.11711195856332779,
+      "rewards/rejected": -1.3533177375793457,
+      "step": 3545
+    },
+    {
+      "epoch": 1.8999832747951162,
+      "grad_norm": 7.006254201358214,
+      "learning_rate": 3.5643261718944346e-07,
+      "logits/chosen": -0.15428760647773743,
+      "logits/rejected": -0.07689885795116425,
+      "logps/chosen": -1.230529546737671,
+      "logps/rejected": -1.3739771842956543,
+      "loss": 1.5886,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.230529546737671,
+      "rewards/margins": 0.14344754815101624,
+      "rewards/rejected": -1.3739771842956543,
+      "step": 3550
+    },
+    {
+      "epoch": 1.902659307576518,
+      "grad_norm": 6.0580380854265705,
+      "learning_rate": 3.5494149530544087e-07,
+      "logits/chosen": -0.27520883083343506,
+      "logits/rejected": -0.15692627429962158,
+      "logps/chosen": -1.1899023056030273,
+      "logps/rejected": -1.3904666900634766,
+      "loss": 1.5944,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.1899023056030273,
+      "rewards/margins": 0.2005644589662552,
+      "rewards/rejected": -1.3904666900634766,
+      "step": 3555
+    },
+    {
+      "epoch": 1.9053353403579194,
+      "grad_norm": 11.666799658627408,
+      "learning_rate": 3.534517807796871e-07,
+      "logits/chosen": -0.16600708663463593,
+      "logits/rejected": -0.10134400427341461,
+      "logps/chosen": -1.282378911972046,
+      "logps/rejected": -1.4656873941421509,
+      "loss": 1.6334,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.282378911972046,
+      "rewards/margins": 0.18330873548984528,
+      "rewards/rejected": -1.4656873941421509,
+      "step": 3560
+    },
+    {
+      "epoch": 1.908011373139321,
+      "grad_norm": 5.999143019086315,
+      "learning_rate": 3.519634880653988e-07,
+      "logits/chosen": -0.18405933678150177,
+      "logits/rejected": -0.14047543704509735,
+      "logps/chosen": -1.2129871845245361,
+      "logps/rejected": -1.4327234029769897,
+      "loss": 1.5922,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.2129871845245361,
+      "rewards/margins": 0.21973614394664764,
+      "rewards/rejected": -1.4327234029769897,
+      "step": 3565
+    },
+    {
+      "epoch": 1.9106874059207226,
+      "grad_norm": 7.594276353635611,
+      "learning_rate": 3.504766316019987e-07,
+      "logits/chosen": -0.2190113514661789,
+      "logits/rejected": -0.10907818377017975,
+      "logps/chosen": -1.241443157196045,
+      "logps/rejected": -1.4419244527816772,
+      "loss": 1.605,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.241443157196045,
+      "rewards/margins": 0.20048122107982635,
+      "rewards/rejected": -1.4419244527816772,
+      "step": 3570
+    },
+    {
+      "epoch": 1.913363438702124,
+      "grad_norm": 7.467008251101331,
+      "learning_rate": 3.489912258149745e-07,
+      "logits/chosen": -0.11534838378429413,
+      "logits/rejected": -0.011506294831633568,
+      "logps/chosen": -1.2071187496185303,
+      "logps/rejected": -1.4129838943481445,
+      "loss": 1.5861,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.2071187496185303,
+      "rewards/margins": 0.20586495101451874,
+      "rewards/rejected": -1.4129838943481445,
+      "step": 3575
+    },
+    {
+      "epoch": 1.9160394714835256,
+      "grad_norm": 8.656615666522297,
+      "learning_rate": 3.475072851157397e-07,
+      "logits/chosen": -0.17539773881435394,
+      "logits/rejected": -0.13611510396003723,
+      "logps/chosen": -1.224860668182373,
+      "logps/rejected": -1.4651345014572144,
+      "loss": 1.578,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.224860668182373,
+      "rewards/margins": 0.24027390778064728,
+      "rewards/rejected": -1.4651345014572144,
+      "step": 3580
+    },
+    {
+      "epoch": 1.9187155042649273,
+      "grad_norm": 8.01258193139656,
+      "learning_rate": 3.460248239014936e-07,
+      "logits/chosen": -0.10515443235635757,
+      "logits/rejected": -0.06597565114498138,
+      "logps/chosen": -1.3152985572814941,
+      "logps/rejected": -1.4554119110107422,
+      "loss": 1.665,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.3152985572814941,
+      "rewards/margins": 0.14011338353157043,
+      "rewards/rejected": -1.4554119110107422,
+      "step": 3585
+    },
+    {
+      "epoch": 1.9213915370463288,
+      "grad_norm": 7.927567069296681,
+      "learning_rate": 3.4454385655508134e-07,
+      "logits/chosen": -0.14241893589496613,
+      "logits/rejected": -0.07710736989974976,
+      "logps/chosen": -1.2774500846862793,
+      "logps/rejected": -1.4123210906982422,
+      "loss": 1.6315,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.2774500846862793,
+      "rewards/margins": 0.13487093150615692,
+      "rewards/rejected": -1.4123210906982422,
+      "step": 3590
+    },
+    {
+      "epoch": 1.9240675698277303,
+      "grad_norm": 6.7613467796414035,
+      "learning_rate": 3.4306439744485447e-07,
+      "logits/chosen": -0.26878923177719116,
+      "logits/rejected": -0.08853518962860107,
+      "logps/chosen": -1.2294676303863525,
+      "logps/rejected": -1.4297891855239868,
+      "loss": 1.5832,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.2294676303863525,
+      "rewards/margins": 0.2003215253353119,
+      "rewards/rejected": -1.4297891855239868,
+      "step": 3595
+    },
+    {
+      "epoch": 1.926743602609132,
+      "grad_norm": 10.040042117098851,
+      "learning_rate": 3.415864609245322e-07,
+      "logits/chosen": -0.12516078352928162,
+      "logits/rejected": 0.03827281668782234,
+      "logps/chosen": -1.2093441486358643,
+      "logps/rejected": -1.4573616981506348,
+      "loss": 1.5646,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.2093441486358643,
+      "rewards/margins": 0.24801769852638245,
+      "rewards/rejected": -1.4573616981506348,
+      "step": 3600
+    },
+    {
+      "epoch": 1.926743602609132,
+      "eval_logits/chosen": 0.0770111232995987,
+      "eval_logits/rejected": 0.1466367095708847,
+      "eval_logps/chosen": -1.2986595630645752,
+      "eval_logps/rejected": -1.477584719657898,
+      "eval_loss": 1.6478548049926758,
+      "eval_rewards/accuracies": 0.5682492852210999,
+      "eval_rewards/chosen": -1.2986595630645752,
+      "eval_rewards/margins": 0.17892497777938843,
+      "eval_rewards/rejected": -1.477584719657898,
+      "eval_runtime": 40.6731,
+      "eval_samples_per_second": 33.069,
+      "eval_steps_per_second": 8.286,
+      "step": 3600
+    },
+    {
+      "epoch": 1.9294196353905335,
+      "grad_norm": 7.840330360879674,
+      "learning_rate": 3.401100613330605e-07,
+      "logits/chosen": -0.22566242516040802,
+      "logits/rejected": -0.19398054480552673,
+      "logps/chosen": -1.2587788105010986,
+      "logps/rejected": -1.3885822296142578,
+      "loss": 1.6406,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.2587788105010986,
+      "rewards/margins": 0.1298035830259323,
+      "rewards/rejected": -1.3885822296142578,
+      "step": 3605
+    },
+    {
+      "epoch": 1.932095668171935,
+      "grad_norm": 6.400210435776477,
+      "learning_rate": 3.3863521299447514e-07,
+      "logits/chosen": -0.15038132667541504,
+      "logits/rejected": -0.04821936413645744,
+      "logps/chosen": -1.2305161952972412,
+      "logps/rejected": -1.4341785907745361,
+      "loss": 1.5762,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.2305161952972412,
+      "rewards/margins": 0.20366230607032776,
+      "rewards/rejected": -1.4341785907745361,
+      "step": 3610
+    },
+    {
+      "epoch": 1.9347717009533367,
+      "grad_norm": 6.307705093571466,
+      "learning_rate": 3.371619302177609e-07,
+      "logits/chosen": -0.0919400304555893,
+      "logits/rejected": 0.0020879185758531094,
+      "logps/chosen": -1.3276575803756714,
+      "logps/rejected": -1.436106562614441,
+      "loss": 1.6671,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.3276575803756714,
+      "rewards/margins": 0.1084490641951561,
+      "rewards/rejected": -1.436106562614441,
+      "step": 3615
+    },
+    {
+      "epoch": 1.9374477337347382,
+      "grad_norm": 10.19933447906074,
+      "learning_rate": 3.3569022729671393e-07,
+      "logits/chosen": -0.14851398766040802,
+      "logits/rejected": -0.09383167326450348,
+      "logps/chosen": -1.2694073915481567,
+      "logps/rejected": -1.383233904838562,
+      "loss": 1.6595,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.2694073915481567,
+      "rewards/margins": 0.11382659524679184,
+      "rewards/rejected": -1.383233904838562,
+      "step": 3620
+    },
+    {
+      "epoch": 1.9401237665161397,
+      "grad_norm": 8.719753915697652,
+      "learning_rate": 3.342201185098024e-07,
+      "logits/chosen": -0.10923445224761963,
+      "logits/rejected": -0.09326710551977158,
+      "logps/chosen": -1.2533637285232544,
+      "logps/rejected": -1.4721083641052246,
+      "loss": 1.601,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.2533637285232544,
+      "rewards/margins": 0.21874471008777618,
+      "rewards/rejected": -1.4721083641052246,
+      "step": 3625
+    },
+    {
+      "epoch": 1.9427997992975414,
+      "grad_norm": 8.66556904039224,
+      "learning_rate": 3.3275161812002807e-07,
+      "logits/chosen": -0.17842338979244232,
+      "logits/rejected": -0.13167932629585266,
+      "logps/chosen": -1.2734184265136719,
+      "logps/rejected": -1.4388821125030518,
+      "loss": 1.6404,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.2734184265136719,
+      "rewards/margins": 0.16546377539634705,
+      "rewards/rejected": -1.4388821125030518,
+      "step": 3630
+    },
+    {
+      "epoch": 1.945475832078943,
+      "grad_norm": 10.758803567850421,
+      "learning_rate": 3.312847403747883e-07,
+      "logits/chosen": -0.20899248123168945,
+      "logits/rejected": -0.13900670409202576,
+      "logps/chosen": -1.2907582521438599,
+      "logps/rejected": -1.4487711191177368,
+      "loss": 1.6511,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.2907582521438599,
+      "rewards/margins": 0.1580127775669098,
+      "rewards/rejected": -1.4487711191177368,
+      "step": 3635
+    },
+    {
+      "epoch": 1.9481518648603444,
+      "grad_norm": 7.904694182249486,
+      "learning_rate": 3.2981949950573733e-07,
+      "logits/chosen": -0.16778028011322021,
+      "logits/rejected": -0.06348295509815216,
+      "logps/chosen": -1.3668639659881592,
+      "logps/rejected": -1.4366734027862549,
+      "loss": 1.7136,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.3668639659881592,
+      "rewards/margins": 0.06980935484170914,
+      "rewards/rejected": -1.4366734027862549,
+      "step": 3640
+    },
+    {
+      "epoch": 1.9508278976417461,
+      "grad_norm": 7.477827845292579,
+      "learning_rate": 3.283559097286486e-07,
+      "logits/chosen": -0.1922161877155304,
+      "logits/rejected": -0.08164320886135101,
+      "logps/chosen": -1.3601343631744385,
+      "logps/rejected": -1.4841011762619019,
+      "loss": 1.6988,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.3601343631744385,
+      "rewards/margins": 0.12396688759326935,
+      "rewards/rejected": -1.4841011762619019,
+      "step": 3645
+    },
+    {
+      "epoch": 1.9535039304231478,
+      "grad_norm": 7.4573408788650175,
+      "learning_rate": 3.268939852432765e-07,
+      "logits/chosen": -0.2014189213514328,
+      "logits/rejected": -0.11250327527523041,
+      "logps/chosen": -1.248978853225708,
+      "logps/rejected": -1.386046290397644,
+      "loss": 1.613,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.248978853225708,
+      "rewards/margins": 0.13706740736961365,
+      "rewards/rejected": -1.386046290397644,
+      "step": 3650
+    },
+    {
+      "epoch": 1.9561799632045491,
+      "grad_norm": 11.245097511626474,
+      "learning_rate": 3.254337402332187e-07,
+      "logits/chosen": -0.1812329888343811,
+      "logits/rejected": -0.060154665261507034,
+      "logps/chosen": -1.3011744022369385,
+      "logps/rejected": -1.4550087451934814,
+      "loss": 1.6674,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.3011744022369385,
+      "rewards/margins": 0.15383443236351013,
+      "rewards/rejected": -1.4550087451934814,
+      "step": 3655
+    },
+    {
+      "epoch": 1.9588559959859508,
+      "grad_norm": 11.108946193918412,
+      "learning_rate": 3.239751888657788e-07,
+      "logits/chosen": -0.21206536889076233,
+      "logits/rejected": -0.10278163105249405,
+      "logps/chosen": -1.2035177946090698,
+      "logps/rejected": -1.3656055927276611,
+      "loss": 1.5971,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.2035177946090698,
+      "rewards/margins": 0.16208767890930176,
+      "rewards/rejected": -1.3656055927276611,
+      "step": 3660
+    },
+    {
+      "epoch": 1.9615320287673526,
+      "grad_norm": 8.840385185457135,
+      "learning_rate": 3.2251834529182856e-07,
+      "logits/chosen": -0.14388488233089447,
+      "logits/rejected": -0.03836951404809952,
+      "logps/chosen": -1.2217673063278198,
+      "logps/rejected": -1.3933287858963013,
+      "loss": 1.6081,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.2217673063278198,
+      "rewards/margins": 0.17156127095222473,
+      "rewards/rejected": -1.3933287858963013,
+      "step": 3665
+    },
+    {
+      "epoch": 1.9642080615487538,
+      "grad_norm": 6.492069858070431,
+      "learning_rate": 3.2106322364567075e-07,
+      "logits/chosen": -0.21870478987693787,
+      "logits/rejected": -0.09473247826099396,
+      "logps/chosen": -1.2585182189941406,
+      "logps/rejected": -1.4683822393417358,
+      "loss": 1.6007,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.2585182189941406,
+      "rewards/margins": 0.20986399054527283,
+      "rewards/rejected": -1.4683822393417358,
+      "step": 3670
+    },
+    {
+      "epoch": 1.9668840943301555,
+      "grad_norm": 7.383800873219924,
+      "learning_rate": 3.1960983804490183e-07,
+      "logits/chosen": -0.16788841784000397,
+      "logits/rejected": -0.04624394327402115,
+      "logps/chosen": -1.2878868579864502,
+      "logps/rejected": -1.57819402217865,
+      "loss": 1.606,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.2878868579864502,
+      "rewards/margins": 0.29030725359916687,
+      "rewards/rejected": -1.57819402217865,
+      "step": 3675
+    },
+    {
+      "epoch": 1.9695601271115573,
+      "grad_norm": 8.130005522098017,
+      "learning_rate": 3.1815820259027537e-07,
+      "logits/chosen": -0.18945178389549255,
+      "logits/rejected": -0.09167564660310745,
+      "logps/chosen": -1.138102412223816,
+      "logps/rejected": -1.3502238988876343,
+      "loss": 1.5201,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.138102412223816,
+      "rewards/margins": 0.21212148666381836,
+      "rewards/rejected": -1.3502238988876343,
+      "step": 3680
+    },
+    {
+      "epoch": 1.9722361598929585,
+      "grad_norm": 8.7096371713661,
+      "learning_rate": 3.16708331365565e-07,
+      "logits/chosen": -0.2038750946521759,
+      "logits/rejected": -0.1391317993402481,
+      "logps/chosen": -1.2368247509002686,
+      "logps/rejected": -1.4675143957138062,
+      "loss": 1.5714,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.2368247509002686,
+      "rewards/margins": 0.23068973422050476,
+      "rewards/rejected": -1.4675143957138062,
+      "step": 3685
+    },
+    {
+      "epoch": 1.9749121926743602,
+      "grad_norm": 7.496384500443867,
+      "learning_rate": 3.152602384374275e-07,
+      "logits/chosen": -0.17525169253349304,
+      "logits/rejected": -0.04767988994717598,
+      "logps/chosen": -1.2744166851043701,
+      "logps/rejected": -1.4640617370605469,
+      "loss": 1.6311,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.2744166851043701,
+      "rewards/margins": 0.18964512646198273,
+      "rewards/rejected": -1.4640617370605469,
+      "step": 3690
+    },
+    {
+      "epoch": 1.977588225455762,
+      "grad_norm": 7.29091326836066,
+      "learning_rate": 3.1381393785526697e-07,
+      "logits/chosen": -0.12062673270702362,
+      "logits/rejected": -0.09473898261785507,
+      "logps/chosen": -1.3177236318588257,
+      "logps/rejected": -1.549801230430603,
+      "loss": 1.6315,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.3177236318588257,
+      "rewards/margins": 0.2320774793624878,
+      "rewards/rejected": -1.549801230430603,
+      "step": 3695
+    },
+    {
+      "epoch": 1.9802642582371635,
+      "grad_norm": 8.520817560244373,
+      "learning_rate": 3.123694436510979e-07,
+      "logits/chosen": -0.12591895461082458,
+      "logits/rejected": -0.035199157893657684,
+      "logps/chosen": -1.2405253648757935,
+      "logps/rejected": -1.4357420206069946,
+      "loss": 1.5927,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.2405253648757935,
+      "rewards/margins": 0.1952165812253952,
+      "rewards/rejected": -1.4357420206069946,
+      "step": 3700
+    },
+    {
+      "epoch": 1.982940291018565,
+      "grad_norm": 8.825227178659631,
+      "learning_rate": 3.1092676983940946e-07,
+      "logits/chosen": -0.16672348976135254,
+      "logits/rejected": -0.10381430387496948,
+      "logps/chosen": -1.290474772453308,
+      "logps/rejected": -1.491180181503296,
+      "loss": 1.6401,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.290474772453308,
+      "rewards/margins": 0.20070545375347137,
+      "rewards/rejected": -1.491180181503296,
+      "step": 3705
+    },
+    {
+      "epoch": 1.9856163237999667,
+      "grad_norm": 7.8779454890869784,
+      "learning_rate": 3.094859304170293e-07,
+      "logits/chosen": -0.04672155901789665,
+      "logits/rejected": 0.0022973031736910343,
+      "logps/chosen": -1.2978581190109253,
+      "logps/rejected": -1.4588404893875122,
+      "loss": 1.6502,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.2978581190109253,
+      "rewards/margins": 0.16098220646381378,
+      "rewards/rejected": -1.4588404893875122,
+      "step": 3710
+    },
+    {
+      "epoch": 1.9882923565813682,
+      "grad_norm": 5.76963985896493,
+      "learning_rate": 3.0804693936298795e-07,
+      "logits/chosen": -0.12356910854578018,
+      "logits/rejected": -0.05897799879312515,
+      "logps/chosen": -1.2731387615203857,
+      "logps/rejected": -1.4750876426696777,
+      "loss": 1.6181,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.2731387615203857,
+      "rewards/margins": 0.201948881149292,
+      "rewards/rejected": -1.4750876426696777,
+      "step": 3715
+    },
+    {
+      "epoch": 1.9909683893627697,
+      "grad_norm": 6.953061439873837,
+      "learning_rate": 3.066098106383826e-07,
+      "logits/chosen": -0.14972397685050964,
+      "logits/rejected": -0.08427880704402924,
+      "logps/chosen": -1.2518134117126465,
+      "logps/rejected": -1.4181315898895264,
+      "loss": 1.6279,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.2518134117126465,
+      "rewards/margins": 0.16631817817687988,
+      "rewards/rejected": -1.4181315898895264,
+      "step": 3720
+    },
+    {
+      "epoch": 1.9936444221441714,
+      "grad_norm": 6.830614461497077,
+      "learning_rate": 3.0517455818624263e-07,
+      "logits/chosen": -0.22917135059833527,
+      "logits/rejected": -0.14272315800189972,
+      "logps/chosen": -1.255526065826416,
+      "logps/rejected": -1.423462152481079,
+      "loss": 1.6155,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.255526065826416,
+      "rewards/margins": 0.16793592274188995,
+      "rewards/rejected": -1.423462152481079,
+      "step": 3725
+    },
+    {
+      "epoch": 1.9963204549255729,
+      "grad_norm": 7.49871108289989,
+      "learning_rate": 3.037411959313936e-07,
+      "logits/chosen": -0.1212211400270462,
+      "logits/rejected": -0.016563747078180313,
+      "logps/chosen": -1.218724012374878,
+      "logps/rejected": -1.4318233728408813,
+      "loss": 1.5747,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.218724012374878,
+      "rewards/margins": 0.21309927105903625,
+      "rewards/rejected": -1.4318233728408813,
+      "step": 3730
+    },
+    {
+      "epoch": 1.9989964877069744,
+      "grad_norm": 7.409246277510923,
+      "learning_rate": 3.023097377803224e-07,
+      "logits/chosen": -0.07228492200374603,
+      "logits/rejected": -0.003942572977393866,
+      "logps/chosen": -1.340967059135437,
+      "logps/rejected": -1.43936026096344,
+      "loss": 1.6969,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.340967059135437,
+      "rewards/margins": 0.09839321672916412,
+      "rewards/rejected": -1.43936026096344,
+      "step": 3735
+    },
+    {
+      "epoch": 2.001672520488376,
+      "grad_norm": 6.988169196963502,
+      "learning_rate": 3.008801976210423e-07,
+      "logits/chosen": -0.0938621237874031,
+      "logits/rejected": -0.05825387313961983,
+      "logps/chosen": -1.3282005786895752,
+      "logps/rejected": -1.4351487159729004,
+      "loss": 1.6719,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.3282005786895752,
+      "rewards/margins": 0.10694797337055206,
+      "rewards/rejected": -1.4351487159729004,
+      "step": 3740
+    },
+    {
+      "epoch": 2.0043485532697773,
+      "grad_norm": 6.621951457816368,
+      "learning_rate": 2.994525893229581e-07,
+      "logits/chosen": -0.17387568950653076,
+      "logits/rejected": -0.09450852870941162,
+      "logps/chosen": -1.275235891342163,
+      "logps/rejected": -1.4434269666671753,
+      "loss": 1.6201,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.275235891342163,
+      "rewards/margins": 0.16819104552268982,
+      "rewards/rejected": -1.4434269666671753,
+      "step": 3745
+    },
+    {
+      "epoch": 2.007024586051179,
+      "grad_norm": 6.744276302001362,
+      "learning_rate": 2.98026926736732e-07,
+      "logits/chosen": -0.21224412322044373,
+      "logits/rejected": -0.14395299553871155,
+      "logps/chosen": -1.1820372343063354,
+      "logps/rejected": -1.4528343677520752,
+      "loss": 1.5254,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.1820372343063354,
+      "rewards/margins": 0.27079716324806213,
+      "rewards/rejected": -1.4528343677520752,
+      "step": 3750
+    },
+    {
+      "epoch": 2.0097006188325808,
+      "grad_norm": 7.532463308561457,
+      "learning_rate": 2.9660322369414846e-07,
+      "logits/chosen": -0.17556189000606537,
+      "logits/rejected": -0.09353695809841156,
+      "logps/chosen": -1.2044193744659424,
+      "logps/rejected": -1.5262267589569092,
+      "loss": 1.5395,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.2044193744659424,
+      "rewards/margins": 0.321807324886322,
+      "rewards/rejected": -1.5262267589569092,
+      "step": 3755
+    },
+    {
+      "epoch": 2.0123766516139825,
+      "grad_norm": 7.32971584019061,
+      "learning_rate": 2.9518149400798063e-07,
+      "logits/chosen": -0.24406211078166962,
+      "logits/rejected": -0.23254680633544922,
+      "logps/chosen": -1.243602991104126,
+      "logps/rejected": -1.4762613773345947,
+      "loss": 1.5733,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.243602991104126,
+      "rewards/margins": 0.23265841603279114,
+      "rewards/rejected": -1.4762613773345947,
+      "step": 3760
+    },
+    {
+      "epoch": 2.0150526843953838,
+      "grad_norm": 8.947032815941157,
+      "learning_rate": 2.9376175147185633e-07,
+      "logits/chosen": -0.10410241782665253,
+      "logits/rejected": 0.04200471192598343,
+      "logps/chosen": -1.2236241102218628,
+      "logps/rejected": -1.4989502429962158,
+      "loss": 1.5573,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.2236241102218628,
+      "rewards/margins": 0.2753261923789978,
+      "rewards/rejected": -1.4989502429962158,
+      "step": 3765
+    },
+    {
+      "epoch": 2.0177287171767855,
+      "grad_norm": 8.738730226260707,
+      "learning_rate": 2.9234400986012376e-07,
+      "logits/chosen": -0.2576754689216614,
+      "logits/rejected": -0.12882332503795624,
+      "logps/chosen": -1.1754076480865479,
+      "logps/rejected": -1.583466649055481,
+      "loss": 1.5065,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.1754076480865479,
+      "rewards/margins": 0.4080590605735779,
+      "rewards/rejected": -1.583466649055481,
+      "step": 3770
+    },
+    {
+      "epoch": 2.020404749958187,
+      "grad_norm": 7.85080913279154,
+      "learning_rate": 2.9092828292771817e-07,
+      "logits/chosen": -0.20311729609966278,
+      "logits/rejected": -0.15802323818206787,
+      "logps/chosen": -1.229494571685791,
+      "logps/rejected": -1.4390629529953003,
+      "loss": 1.5964,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.229494571685791,
+      "rewards/margins": 0.20956823229789734,
+      "rewards/rejected": -1.4390629529953003,
+      "step": 3775
+    },
+    {
+      "epoch": 2.0230807827395885,
+      "grad_norm": 7.165995730205779,
+      "learning_rate": 2.8951458441002875e-07,
+      "logits/chosen": -0.13184405863285065,
+      "logits/rejected": -0.1076413244009018,
+      "logps/chosen": -1.2524865865707397,
+      "logps/rejected": -1.4842063188552856,
+      "loss": 1.6219,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.2524865865707397,
+      "rewards/margins": 0.2317197024822235,
+      "rewards/rejected": -1.4842063188552856,
+      "step": 3780
+    },
+    {
+      "epoch": 2.02575681552099,
+      "grad_norm": 6.470693358098353,
+      "learning_rate": 2.881029280227643e-07,
+      "logits/chosen": -0.18296106159687042,
+      "logits/rejected": -0.0861835852265358,
+      "logps/chosen": -1.2711610794067383,
+      "logps/rejected": -1.5663596391677856,
+      "loss": 1.5752,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.2711610794067383,
+      "rewards/margins": 0.295198529958725,
+      "rewards/rejected": -1.5663596391677856,
+      "step": 3785
+    },
+    {
+      "epoch": 2.028432848302392,
+      "grad_norm": 6.194312588328929,
+      "learning_rate": 2.8669332746182177e-07,
+      "logits/chosen": -0.2417146861553192,
+      "logits/rejected": -0.0933353453874588,
+      "logps/chosen": -1.2275288105010986,
+      "logps/rejected": -1.4598970413208008,
+      "loss": 1.556,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.2275288105010986,
+      "rewards/margins": 0.23236839473247528,
+      "rewards/rejected": -1.4598970413208008,
+      "step": 3790
+    },
+    {
+      "epoch": 2.031108881083793,
+      "grad_norm": 6.4618508390800615,
+      "learning_rate": 2.8528579640315156e-07,
+      "logits/chosen": -0.1630270779132843,
+      "logits/rejected": -0.14098826050758362,
+      "logps/chosen": -1.2046821117401123,
+      "logps/rejected": -1.3948853015899658,
+      "loss": 1.5704,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.2046821117401123,
+      "rewards/margins": 0.19020314514636993,
+      "rewards/rejected": -1.3948853015899658,
+      "step": 3795
+    },
+    {
+      "epoch": 2.033784913865195,
+      "grad_norm": 8.434133849723032,
+      "learning_rate": 2.8388034850262646e-07,
+      "logits/chosen": -0.15991130471229553,
+      "logits/rejected": -0.059218693524599075,
+      "logps/chosen": -1.2866146564483643,
+      "logps/rejected": -1.5523066520690918,
+      "loss": 1.6305,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.2866146564483643,
+      "rewards/margins": 0.2656919062137604,
+      "rewards/rejected": -1.5523066520690918,
+      "step": 3800
+    },
+    {
+      "epoch": 2.0364609466465966,
+      "grad_norm": 11.017392314905319,
+      "learning_rate": 2.824769973959079e-07,
+      "logits/chosen": -0.12081266939640045,
+      "logits/rejected": -0.023985737934708595,
+      "logps/chosen": -1.1901729106903076,
+      "logps/rejected": -1.4317840337753296,
+      "loss": 1.5337,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.1901729106903076,
+      "rewards/margins": 0.24161119759082794,
+      "rewards/rejected": -1.4317840337753296,
+      "step": 3805
+    },
+    {
+      "epoch": 2.039136979427998,
+      "grad_norm": 7.846576772195456,
+      "learning_rate": 2.81075756698315e-07,
+      "logits/chosen": -0.047540389001369476,
+      "logits/rejected": 0.03174520656466484,
+      "logps/chosen": -1.2199788093566895,
+      "logps/rejected": -1.4722950458526611,
+      "loss": 1.5653,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.2199788093566895,
+      "rewards/margins": 0.25231608748435974,
+      "rewards/rejected": -1.4722950458526611,
+      "step": 3810
+    },
+    {
+      "epoch": 2.0418130122093996,
+      "grad_norm": 7.380796979122265,
+      "learning_rate": 2.7967664000469035e-07,
+      "logits/chosen": -0.25073981285095215,
+      "logits/rejected": -0.14145082235336304,
+      "logps/chosen": -1.259158968925476,
+      "logps/rejected": -1.4133001565933228,
+      "loss": 1.6182,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.259158968925476,
+      "rewards/margins": 0.15414123237133026,
+      "rewards/rejected": -1.4133001565933228,
+      "step": 3815
+    },
+    {
+      "epoch": 2.0444890449908013,
+      "grad_norm": 9.109162210572233,
+      "learning_rate": 2.7827966088927095e-07,
+      "logits/chosen": -0.23392069339752197,
+      "logits/rejected": -0.06347467750310898,
+      "logps/chosen": -1.3198875188827515,
+      "logps/rejected": -1.4128139019012451,
+      "loss": 1.6813,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -1.3198875188827515,
+      "rewards/margins": 0.09292645007371902,
+      "rewards/rejected": -1.4128139019012451,
+      "step": 3820
+    },
+    {
+      "epoch": 2.0471650777722026,
+      "grad_norm": 8.398550301698064,
+      "learning_rate": 2.768848329055538e-07,
+      "logits/chosen": -0.20006170868873596,
+      "logits/rejected": -0.10423286259174347,
+      "logps/chosen": -1.228200078010559,
+      "logps/rejected": -1.374854326248169,
+      "loss": 1.5936,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.228200078010559,
+      "rewards/margins": 0.14665424823760986,
+      "rewards/rejected": -1.374854326248169,
+      "step": 3825
+    },
+    {
+      "epoch": 2.0498411105536043,
+      "grad_norm": 8.515263228939604,
+      "learning_rate": 2.7549216958616657e-07,
+      "logits/chosen": -0.24098920822143555,
+      "logits/rejected": -0.11884365230798721,
+      "logps/chosen": -1.2912288904190063,
+      "logps/rejected": -1.5368410348892212,
+      "loss": 1.6143,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.2912288904190063,
+      "rewards/margins": 0.24561218917369843,
+      "rewards/rejected": -1.5368410348892212,
+      "step": 3830
+    },
+    {
+      "epoch": 2.052517143335006,
+      "grad_norm": 7.409912246633359,
+      "learning_rate": 2.741016844427344e-07,
+      "logits/chosen": -0.15882329642772675,
+      "logits/rejected": -0.04877109453082085,
+      "logps/chosen": -1.2781291007995605,
+      "logps/rejected": -1.4769856929779053,
+      "loss": 1.6229,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.2781291007995605,
+      "rewards/margins": 0.1988566666841507,
+      "rewards/rejected": -1.4769856929779053,
+      "step": 3835
+    },
+    {
+      "epoch": 2.0551931761164073,
+      "grad_norm": 8.036877785952855,
+      "learning_rate": 2.7271339096575073e-07,
+      "logits/chosen": -0.1380457580089569,
+      "logits/rejected": -0.06133418530225754,
+      "logps/chosen": -1.1802645921707153,
+      "logps/rejected": -1.5075302124023438,
+      "loss": 1.5241,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.1802645921707153,
+      "rewards/margins": 0.3272656500339508,
+      "rewards/rejected": -1.5075302124023438,
+      "step": 3840
+    },
+    {
+      "epoch": 2.057869208897809,
+      "grad_norm": 7.1543336062609955,
+      "learning_rate": 2.713273026244446e-07,
+      "logits/chosen": -0.27739790081977844,
+      "logits/rejected": -0.07038364559412003,
+      "logps/chosen": -1.3000078201293945,
+      "logps/rejected": -1.5286756753921509,
+      "loss": 1.6336,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.3000078201293945,
+      "rewards/margins": 0.22866801917552948,
+      "rewards/rejected": -1.5286756753921509,
+      "step": 3845
+    },
+    {
+      "epoch": 2.0605452416792107,
+      "grad_norm": 7.797180213118202,
+      "learning_rate": 2.6994343286665156e-07,
+      "logits/chosen": -0.18627727031707764,
+      "logits/rejected": -0.04998369142413139,
+      "logps/chosen": -1.2930742502212524,
+      "logps/rejected": -1.4864602088928223,
+      "loss": 1.6311,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.2930742502212524,
+      "rewards/margins": 0.1933860182762146,
+      "rewards/rejected": -1.4864602088928223,
+      "step": 3850
+    },
+    {
+      "epoch": 2.063221274460612,
+      "grad_norm": 7.29308685777475,
+      "learning_rate": 2.6856179511868156e-07,
+      "logits/chosen": -0.14326825737953186,
+      "logits/rejected": 0.013318806886672974,
+      "logps/chosen": -1.2368152141571045,
+      "logps/rejected": -1.5324199199676514,
+      "loss": 1.5788,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.2368152141571045,
+      "rewards/margins": 0.2956047058105469,
+      "rewards/rejected": -1.5324199199676514,
+      "step": 3855
+    },
+    {
+      "epoch": 2.0658973072420137,
+      "grad_norm": 6.185287692033218,
+      "learning_rate": 2.6718240278519056e-07,
+      "logits/chosen": -0.14798951148986816,
+      "logits/rejected": -0.008536259643733501,
+      "logps/chosen": -1.2535607814788818,
+      "logps/rejected": -1.4576365947723389,
+      "loss": 1.6073,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.2535607814788818,
+      "rewards/margins": 0.20407557487487793,
+      "rewards/rejected": -1.4576365947723389,
+      "step": 3860
+    },
+    {
+      "epoch": 2.0685733400234154,
+      "grad_norm": 10.875993443912234,
+      "learning_rate": 2.6580526924904866e-07,
+      "logits/chosen": -0.29120516777038574,
+      "logits/rejected": -0.135166734457016,
+      "logps/chosen": -1.2904469966888428,
+      "logps/rejected": -1.4185327291488647,
+      "loss": 1.6384,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.2904469966888428,
+      "rewards/margins": 0.1280859410762787,
+      "rewards/rejected": -1.4185327291488647,
+      "step": 3865
+    },
+    {
+      "epoch": 2.0712493728048167,
+      "grad_norm": 9.152730161588952,
+      "learning_rate": 2.6443040787121186e-07,
+      "logits/chosen": -0.19847515225410461,
+      "logits/rejected": -0.1773093342781067,
+      "logps/chosen": -1.1396772861480713,
+      "logps/rejected": -1.3321276903152466,
+      "loss": 1.5195,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.1396772861480713,
+      "rewards/margins": 0.19245044887065887,
+      "rewards/rejected": -1.3321276903152466,
+      "step": 3870
+    },
+    {
+      "epoch": 2.0739254055862184,
+      "grad_norm": 6.883716697989702,
+      "learning_rate": 2.6305783199059084e-07,
+      "logits/chosen": -0.19664858281612396,
+      "logits/rejected": -0.09270669519901276,
+      "logps/chosen": -1.2300426959991455,
+      "logps/rejected": -1.4747213125228882,
+      "loss": 1.568,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.2300426959991455,
+      "rewards/margins": 0.2446787804365158,
+      "rewards/rejected": -1.4747213125228882,
+      "step": 3875
+    },
+    {
+      "epoch": 2.07660143836762,
+      "grad_norm": 6.804884424226387,
+      "learning_rate": 2.6168755492392324e-07,
+      "logits/chosen": -0.21317140758037567,
+      "logits/rejected": -0.08506114780902863,
+      "logps/chosen": -1.1500242948532104,
+      "logps/rejected": -1.3966689109802246,
+      "loss": 1.4991,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.1500242948532104,
+      "rewards/margins": 0.24664464592933655,
+      "rewards/rejected": -1.3966689109802246,
+      "step": 3880
+    },
+    {
+      "epoch": 2.0792774711490214,
+      "grad_norm": 7.374519752982465,
+      "learning_rate": 2.6031958996564274e-07,
+      "logits/chosen": -0.22471275925636292,
+      "logits/rejected": -0.09944645315408707,
+      "logps/chosen": -1.1963858604431152,
+      "logps/rejected": -1.4682543277740479,
+      "loss": 1.533,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.1963858604431152,
+      "rewards/margins": 0.27186843752861023,
+      "rewards/rejected": -1.4682543277740479,
+      "step": 3885
+    },
+    {
+      "epoch": 2.081953503930423,
+      "grad_norm": 10.671471040265736,
+      "learning_rate": 2.589539503877518e-07,
+      "logits/chosen": -0.14481914043426514,
+      "logits/rejected": -0.06979800015687943,
+      "logps/chosen": -1.2372180223464966,
+      "logps/rejected": -1.419391393661499,
+      "loss": 1.6031,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.2372180223464966,
+      "rewards/margins": 0.18217317759990692,
+      "rewards/rejected": -1.419391393661499,
+      "step": 3890
+    },
+    {
+      "epoch": 2.084629536711825,
+      "grad_norm": 9.820407721904644,
+      "learning_rate": 2.5759064943969125e-07,
+      "logits/chosen": -0.18728923797607422,
+      "logits/rejected": 0.020716819912195206,
+      "logps/chosen": -1.2106778621673584,
+      "logps/rejected": -1.4359009265899658,
+      "loss": 1.5687,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.2106778621673584,
+      "rewards/margins": 0.2252231389284134,
+      "rewards/rejected": -1.4359009265899658,
+      "step": 3895
+    },
+    {
+      "epoch": 2.087305569493226,
+      "grad_norm": 8.506512351210667,
+      "learning_rate": 2.562297003482131e-07,
+      "logits/chosen": -0.08664155006408691,
+      "logits/rejected": -0.08900171518325806,
+      "logps/chosen": -1.216568946838379,
+      "logps/rejected": -1.4304020404815674,
+      "loss": 1.5729,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.216568946838379,
+      "rewards/margins": 0.2138332575559616,
+      "rewards/rejected": -1.4304020404815674,
+      "step": 3900
+    },
+    {
+      "epoch": 2.089981602274628,
+      "grad_norm": 7.499504238653955,
+      "learning_rate": 2.548711163172512e-07,
+      "logits/chosen": -0.1457064300775528,
+      "logits/rejected": -0.05645012855529785,
+      "logps/chosen": -1.2313451766967773,
+      "logps/rejected": -1.4008392095565796,
+      "loss": 1.5811,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.2313451766967773,
+      "rewards/margins": 0.16949406266212463,
+      "rewards/rejected": -1.4008392095565796,
+      "step": 3905
+    },
+    {
+      "epoch": 2.0926576350560295,
+      "grad_norm": 6.809512264296725,
+      "learning_rate": 2.53514910527794e-07,
+      "logits/chosen": -0.10277318954467773,
+      "logits/rejected": -0.003941692411899567,
+      "logps/chosen": -1.1524231433868408,
+      "logps/rejected": -1.392700433731079,
+      "loss": 1.5247,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.1524231433868408,
+      "rewards/margins": 0.24027732014656067,
+      "rewards/rejected": -1.392700433731079,
+      "step": 3910
+    },
+    {
+      "epoch": 2.095333667837431,
+      "grad_norm": 5.704136859915198,
+      "learning_rate": 2.5216109613775573e-07,
+      "logits/chosen": -0.189134418964386,
+      "logits/rejected": -0.05074596405029297,
+      "logps/chosen": -1.2564955949783325,
+      "logps/rejected": -1.5363318920135498,
+      "loss": 1.5776,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.2564955949783325,
+      "rewards/margins": 0.2798364758491516,
+      "rewards/rejected": -1.5363318920135498,
+      "step": 3915
+    },
+    {
+      "epoch": 2.0980097006188325,
+      "grad_norm": 6.740453561840508,
+      "learning_rate": 2.5080968628184993e-07,
+      "logits/chosen": -0.19768080115318298,
+      "logits/rejected": -0.05966009944677353,
+      "logps/chosen": -1.265911340713501,
+      "logps/rejected": -1.5537766218185425,
+      "loss": 1.5768,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.265911340713501,
+      "rewards/margins": 0.28786543011665344,
+      "rewards/rejected": -1.5537766218185425,
+      "step": 3920
+    },
+    {
+      "epoch": 2.1006857334002342,
+      "grad_norm": 7.8854205875236625,
+      "learning_rate": 2.494606940714605e-07,
+      "logits/chosen": -0.1744232028722763,
+      "logits/rejected": -0.07103218138217926,
+      "logps/chosen": -1.1927406787872314,
+      "logps/rejected": -1.4526997804641724,
+      "loss": 1.5685,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.1927406787872314,
+      "rewards/margins": 0.25995901226997375,
+      "rewards/rejected": -1.4526997804641724,
+      "step": 3925
+    },
+    {
+      "epoch": 2.103361766181636,
+      "grad_norm": 6.305959398254729,
+      "learning_rate": 2.4811413259451625e-07,
+      "logits/chosen": -0.25274330377578735,
+      "logits/rejected": -0.13995765149593353,
+      "logps/chosen": -1.2792290449142456,
+      "logps/rejected": -1.4351447820663452,
+      "loss": 1.6262,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.2792290449142456,
+      "rewards/margins": 0.15591581165790558,
+      "rewards/rejected": -1.4351447820663452,
+      "step": 3930
+    },
+    {
+      "epoch": 2.106037798963037,
+      "grad_norm": 7.203853391762811,
+      "learning_rate": 2.46770014915362e-07,
+      "logits/chosen": -0.09655294567346573,
+      "logits/rejected": -0.04600976034998894,
+      "logps/chosen": -1.2321531772613525,
+      "logps/rejected": -1.5000993013381958,
+      "loss": 1.5579,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.2321531772613525,
+      "rewards/margins": 0.26794594526290894,
+      "rewards/rejected": -1.5000993013381958,
+      "step": 3935
+    },
+    {
+      "epoch": 2.108713831744439,
+      "grad_norm": 9.213519824001787,
+      "learning_rate": 2.45428354074634e-07,
+      "logits/chosen": -0.13530853390693665,
+      "logits/rejected": -0.10061784088611603,
+      "logps/chosen": -1.1457960605621338,
+      "logps/rejected": -1.5235750675201416,
+      "loss": 1.4669,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.1457960605621338,
+      "rewards/margins": 0.37777888774871826,
+      "rewards/rejected": -1.5235750675201416,
+      "step": 3940
+    },
+    {
+      "epoch": 2.1113898645258407,
+      "grad_norm": 7.951658906871399,
+      "learning_rate": 2.4408916308913105e-07,
+      "logits/chosen": -0.15835228562355042,
+      "logits/rejected": -0.006830123718827963,
+      "logps/chosen": -1.276841402053833,
+      "logps/rejected": -1.4330775737762451,
+      "loss": 1.631,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.276841402053833,
+      "rewards/margins": 0.15623629093170166,
+      "rewards/rejected": -1.4330775737762451,
+      "step": 3945
+    },
+    {
+      "epoch": 2.114065897307242,
+      "grad_norm": 9.809389717974016,
+      "learning_rate": 2.4275245495169025e-07,
+      "logits/chosen": -0.08743122965097427,
+      "logits/rejected": 0.04935204237699509,
+      "logps/chosen": -1.2254064083099365,
+      "logps/rejected": -1.3818477392196655,
+      "loss": 1.6021,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.2254064083099365,
+      "rewards/margins": 0.15644130110740662,
+      "rewards/rejected": -1.3818477392196655,
+      "step": 3950
+    },
+    {
+      "epoch": 2.1167419300886436,
+      "grad_norm": 7.760483638978795,
+      "learning_rate": 2.414182426310597e-07,
+      "logits/chosen": -0.240851491689682,
+      "logits/rejected": -0.16412867605686188,
+      "logps/chosen": -1.1903098821640015,
+      "logps/rejected": -1.4226571321487427,
+      "loss": 1.5391,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.1903098821640015,
+      "rewards/margins": 0.23234716057777405,
+      "rewards/rejected": -1.4226571321487427,
+      "step": 3955
+    },
+    {
+      "epoch": 2.1194179628700454,
+      "grad_norm": 11.898354640392771,
+      "learning_rate": 2.400865390717734e-07,
+      "logits/chosen": -0.1388021856546402,
+      "logits/rejected": -0.03632751852273941,
+      "logps/chosen": -1.2601227760314941,
+      "logps/rejected": -1.5602470636367798,
+      "loss": 1.5906,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.2601227760314941,
+      "rewards/margins": 0.3001241087913513,
+      "rewards/rejected": -1.5602470636367798,
+      "step": 3960
+    },
+    {
+      "epoch": 2.1220939956514466,
+      "grad_norm": 7.148422792573522,
+      "learning_rate": 2.3875735719402475e-07,
+      "logits/chosen": -0.08727221935987473,
+      "logits/rejected": 0.004704681225121021,
+      "logps/chosen": -1.1741969585418701,
+      "logps/rejected": -1.4731131792068481,
+      "loss": 1.5135,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.1741969585418701,
+      "rewards/margins": 0.29891616106033325,
+      "rewards/rejected": -1.4731131792068481,
+      "step": 3965
+    },
+    {
+      "epoch": 2.1247700284328483,
+      "grad_norm": 7.22524858929848,
+      "learning_rate": 2.3743070989354258e-07,
+      "logits/chosen": -0.16493722796440125,
+      "logits/rejected": -0.09080357104539871,
+      "logps/chosen": -1.2232723236083984,
+      "logps/rejected": -1.5072360038757324,
+      "loss": 1.5719,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.2232723236083984,
+      "rewards/margins": 0.28396373987197876,
+      "rewards/rejected": -1.5072360038757324,
+      "step": 3970
+    },
+    {
+      "epoch": 2.12744606121425,
+      "grad_norm": 6.966234959523901,
+      "learning_rate": 2.3610661004146454e-07,
+      "logits/chosen": -0.09802804887294769,
+      "logits/rejected": -0.02745756134390831,
+      "logps/chosen": -1.1443368196487427,
+      "logps/rejected": -1.40862238407135,
+      "loss": 1.5115,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.1443368196487427,
+      "rewards/margins": 0.26428547501564026,
+      "rewards/rejected": -1.40862238407135,
+      "step": 3975
+    },
+    {
+      "epoch": 2.1301220939956513,
+      "grad_norm": 8.690475995299611,
+      "learning_rate": 2.3478507048421314e-07,
+      "logits/chosen": -0.21995559334754944,
+      "logits/rejected": -0.16529780626296997,
+      "logps/chosen": -1.142746925354004,
+      "logps/rejected": -1.4897350072860718,
+      "loss": 1.4869,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.142746925354004,
+      "rewards/margins": 0.34698814153671265,
+      "rewards/rejected": -1.4897350072860718,
+      "step": 3980
+    },
+    {
+      "epoch": 2.132798126777053,
+      "grad_norm": 11.365330215938014,
+      "learning_rate": 2.334661040433713e-07,
+      "logits/chosen": -0.24035818874835968,
+      "logits/rejected": -0.15376703441143036,
+      "logps/chosen": -1.217773199081421,
+      "logps/rejected": -1.4895384311676025,
+      "loss": 1.5593,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.217773199081421,
+      "rewards/margins": 0.27176520228385925,
+      "rewards/rejected": -1.4895384311676025,
+      "step": 3985
+    },
+    {
+      "epoch": 2.1354741595584548,
+      "grad_norm": 7.236942359737125,
+      "learning_rate": 2.321497235155568e-07,
+      "logits/chosen": -0.2544265687465668,
+      "logits/rejected": -0.1432047337293625,
+      "logps/chosen": -1.1897337436676025,
+      "logps/rejected": -1.4237349033355713,
+      "loss": 1.5348,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.1897337436676025,
+      "rewards/margins": 0.23400099575519562,
+      "rewards/rejected": -1.4237349033355713,
+      "step": 3990
+    },
+    {
+      "epoch": 2.138150192339856,
+      "grad_norm": 7.941620296898851,
+      "learning_rate": 2.3083594167229965e-07,
+      "logits/chosen": -0.302916556596756,
+      "logits/rejected": -0.0914468914270401,
+      "logps/chosen": -1.2181164026260376,
+      "logps/rejected": -1.5334707498550415,
+      "loss": 1.5571,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.2181164026260376,
+      "rewards/margins": 0.3153544068336487,
+      "rewards/rejected": -1.5334707498550415,
+      "step": 3995
+    },
+    {
+      "epoch": 2.1408262251212578,
+      "grad_norm": 9.043760190413138,
+      "learning_rate": 2.295247712599167e-07,
+      "logits/chosen": -0.19980423152446747,
+      "logits/rejected": -0.10652852058410645,
+      "logps/chosen": -1.19603431224823,
+      "logps/rejected": -1.461136817932129,
+      "loss": 1.5351,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.19603431224823,
+      "rewards/margins": 0.2651025950908661,
+      "rewards/rejected": -1.461136817932129,
+      "step": 4000
+    },
+    {
+      "epoch": 2.1408262251212578,
+      "eval_logits/chosen": 0.07140856981277466,
+      "eval_logits/rejected": 0.1418316513299942,
+      "eval_logps/chosen": -1.3020397424697876,
+      "eval_logps/rejected": -1.496042013168335,
+      "eval_loss": 1.6470303535461426,
+      "eval_rewards/accuracies": 0.5697329640388489,
+      "eval_rewards/chosen": -1.3020397424697876,
+      "eval_rewards/margins": 0.19400210678577423,
+      "eval_rewards/rejected": -1.496042013168335,
+      "eval_runtime": 40.7989,
+      "eval_samples_per_second": 32.967,
+      "eval_steps_per_second": 8.26,
+      "step": 4000
+    },
+    {
+      "epoch": 2.1435022579026595,
+      "grad_norm": 6.69689106524811,
+      "learning_rate": 2.2821622499938948e-07,
+      "logits/chosen": -0.19220581650733948,
+      "logits/rejected": -0.00428193062543869,
+      "logps/chosen": -1.3547049760818481,
+      "logps/rejected": -1.513340950012207,
+      "loss": 1.6739,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.3547049760818481,
+      "rewards/margins": 0.15863582491874695,
+      "rewards/rejected": -1.513340950012207,
+      "step": 4005
+    },
+    {
+      "epoch": 2.1461782906840607,
+      "grad_norm": 6.439031174437564,
+      "learning_rate": 2.269103155862391e-07,
+      "logits/chosen": -0.2250560224056244,
+      "logits/rejected": -0.14797084033489227,
+      "logps/chosen": -1.2645237445831299,
+      "logps/rejected": -1.4367012977600098,
+      "loss": 1.6099,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.2645237445831299,
+      "rewards/margins": 0.1721775382757187,
+      "rewards/rejected": -1.4367012977600098,
+      "step": 4010
+    },
+    {
+      "epoch": 2.1488543234654625,
+      "grad_norm": 7.528147440207392,
+      "learning_rate": 2.2560705569040483e-07,
+      "logits/chosen": -0.2006227672100067,
+      "logits/rejected": 0.018616771325469017,
+      "logps/chosen": -1.2492566108703613,
+      "logps/rejected": -1.4439423084259033,
+      "loss": 1.6014,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.2492566108703613,
+      "rewards/margins": 0.19468554854393005,
+      "rewards/rejected": -1.4439423084259033,
+      "step": 4015
+    },
+    {
+      "epoch": 2.151530356246864,
+      "grad_norm": 6.76322281964501,
+      "learning_rate": 2.2430645795611963e-07,
+      "logits/chosen": -0.27711382508277893,
+      "logits/rejected": -0.16055673360824585,
+      "logps/chosen": -1.2736780643463135,
+      "logps/rejected": -1.437705636024475,
+      "loss": 1.6194,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.2736780643463135,
+      "rewards/margins": 0.16402749717235565,
+      "rewards/rejected": -1.437705636024475,
+      "step": 4020
+    },
+    {
+      "epoch": 2.1542063890282654,
+      "grad_norm": 8.161004313533256,
+      "learning_rate": 2.230085350017884e-07,
+      "logits/chosen": -0.20443399250507355,
+      "logits/rejected": -0.12384285032749176,
+      "logps/chosen": -1.1757631301879883,
+      "logps/rejected": -1.4113733768463135,
+      "loss": 1.5517,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.1757631301879883,
+      "rewards/margins": 0.23561033606529236,
+      "rewards/rejected": -1.4113733768463135,
+      "step": 4025
+    },
+    {
+      "epoch": 2.156882421809667,
+      "grad_norm": 12.639231430085822,
+      "learning_rate": 2.2171329941986554e-07,
+      "logits/chosen": -0.23632891476154327,
+      "logits/rejected": -0.17125725746154785,
+      "logps/chosen": -1.194329857826233,
+      "logps/rejected": -1.4643354415893555,
+      "loss": 1.5404,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.194329857826233,
+      "rewards/margins": 0.27000561356544495,
+      "rewards/rejected": -1.4643354415893555,
+      "step": 4030
+    },
+    {
+      "epoch": 2.159558454591069,
+      "grad_norm": 13.035082920965463,
+      "learning_rate": 2.2042076377673202e-07,
+      "logits/chosen": -0.22286994755268097,
+      "logits/rejected": -0.19176195561885834,
+      "logps/chosen": -1.2081761360168457,
+      "logps/rejected": -1.312530755996704,
+      "loss": 1.6095,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.2081761360168457,
+      "rewards/margins": 0.10435459762811661,
+      "rewards/rejected": -1.312530755996704,
+      "step": 4035
+    },
+    {
+      "epoch": 2.16223448737247,
+      "grad_norm": 6.83123896732779,
+      "learning_rate": 2.1913094061257476e-07,
+      "logits/chosen": -0.22944974899291992,
+      "logits/rejected": -0.19934967160224915,
+      "logps/chosen": -1.1782562732696533,
+      "logps/rejected": -1.3568545579910278,
+      "loss": 1.55,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.1782562732696533,
+      "rewards/margins": 0.1785982847213745,
+      "rewards/rejected": -1.3568545579910278,
+      "step": 4040
+    },
+    {
+      "epoch": 2.164910520153872,
+      "grad_norm": 9.180685223568698,
+      "learning_rate": 2.178438424412633e-07,
+      "logits/chosen": -0.16974474489688873,
+      "logits/rejected": -0.06564159691333771,
+      "logps/chosen": -1.2702267169952393,
+      "logps/rejected": -1.4524881839752197,
+      "loss": 1.6181,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.2702267169952393,
+      "rewards/margins": 0.18226134777069092,
+      "rewards/rejected": -1.4524881839752197,
+      "step": 4045
+    },
+    {
+      "epoch": 2.1675865529352736,
+      "grad_norm": 4.982141323452237,
+      "learning_rate": 2.165594817502302e-07,
+      "logits/chosen": -0.24664337933063507,
+      "logits/rejected": -0.16238948702812195,
+      "logps/chosen": -1.255029559135437,
+      "logps/rejected": -1.4013950824737549,
+      "loss": 1.6247,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.255029559135437,
+      "rewards/margins": 0.14636535942554474,
+      "rewards/rejected": -1.4013950824737549,
+      "step": 4050
+    },
+    {
+      "epoch": 2.170262585716675,
+      "grad_norm": 6.9593790107088855,
+      "learning_rate": 2.1527787100034806e-07,
+      "logits/chosen": -0.1397363841533661,
+      "logits/rejected": -0.09621234238147736,
+      "logps/chosen": -1.23110032081604,
+      "logps/rejected": -1.4188356399536133,
+      "loss": 1.5766,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.23110032081604,
+      "rewards/margins": 0.1877353936433792,
+      "rewards/rejected": -1.4188356399536133,
+      "step": 4055
+    },
+    {
+      "epoch": 2.1729386184980766,
+      "grad_norm": 8.771723975407568,
+      "learning_rate": 2.1399902262581037e-07,
+      "logits/chosen": -0.047494079917669296,
+      "logits/rejected": 0.030322402715682983,
+      "logps/chosen": -1.1686232089996338,
+      "logps/rejected": -1.3430014848709106,
+      "loss": 1.538,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.1686232089996338,
+      "rewards/margins": 0.17437830567359924,
+      "rewards/rejected": -1.3430014848709106,
+      "step": 4060
+    },
+    {
+      "epoch": 2.1756146512794783,
+      "grad_norm": 8.165202393914798,
+      "learning_rate": 2.127229490340094e-07,
+      "logits/chosen": -0.2657929062843323,
+      "logits/rejected": -0.19261549413204193,
+      "logps/chosen": -1.2344993352890015,
+      "logps/rejected": -1.4854099750518799,
+      "loss": 1.557,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.2344993352890015,
+      "rewards/margins": 0.2509106397628784,
+      "rewards/rejected": -1.4854099750518799,
+      "step": 4065
+    },
+    {
+      "epoch": 2.1782906840608796,
+      "grad_norm": 11.282347383089665,
+      "learning_rate": 2.1144966260541698e-07,
+      "logits/chosen": -0.147164449095726,
+      "logits/rejected": 0.02844533883035183,
+      "logps/chosen": -1.193411111831665,
+      "logps/rejected": -1.4447523355484009,
+      "loss": 1.542,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.193411111831665,
+      "rewards/margins": 0.25134122371673584,
+      "rewards/rejected": -1.4447523355484009,
+      "step": 4070
+    },
+    {
+      "epoch": 2.1809667168422813,
+      "grad_norm": 7.959387334694807,
+      "learning_rate": 2.1017917569346332e-07,
+      "logits/chosen": -0.19362321496009827,
+      "logits/rejected": -0.038269199430942535,
+      "logps/chosen": -1.2862924337387085,
+      "logps/rejected": -1.4655495882034302,
+      "loss": 1.6203,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.2862924337387085,
+      "rewards/margins": 0.17925693094730377,
+      "rewards/rejected": -1.4655495882034302,
+      "step": 4075
+    },
+    {
+      "epoch": 2.183642749623683,
+      "grad_norm": 5.997204567632148,
+      "learning_rate": 2.0891150062441837e-07,
+      "logits/chosen": -0.19443969428539276,
+      "logits/rejected": -0.08346713334321976,
+      "logps/chosen": -1.2668898105621338,
+      "logps/rejected": -1.4902750253677368,
+      "loss": 1.6037,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.2668898105621338,
+      "rewards/margins": 0.22338524460792542,
+      "rewards/rejected": -1.4902750253677368,
+      "step": 4080
+    },
+    {
+      "epoch": 2.1863187824050843,
+      "grad_norm": 7.002594484782536,
+      "learning_rate": 2.0764664969727086e-07,
+      "logits/chosen": -0.1699420064687729,
+      "logits/rejected": -0.0828448086977005,
+      "logps/chosen": -1.2525122165679932,
+      "logps/rejected": -1.4004987478256226,
+      "loss": 1.6426,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.2525122165679932,
+      "rewards/margins": 0.147986501455307,
+      "rewards/rejected": -1.4004987478256226,
+      "step": 4085
+    },
+    {
+      "epoch": 2.188994815186486,
+      "grad_norm": 6.749307057443181,
+      "learning_rate": 2.0638463518361033e-07,
+      "logits/chosen": -0.23159702122211456,
+      "logits/rejected": -0.07271578907966614,
+      "logps/chosen": -1.2239384651184082,
+      "logps/rejected": -1.4089152812957764,
+      "loss": 1.5835,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.2239384651184082,
+      "rewards/margins": 0.18497692048549652,
+      "rewards/rejected": -1.4089152812957764,
+      "step": 4090
+    },
+    {
+      "epoch": 2.1916708479678877,
+      "grad_norm": 8.019662305992837,
+      "learning_rate": 2.0512546932750702e-07,
+      "logits/chosen": -0.2117147445678711,
+      "logits/rejected": -0.14385417103767395,
+      "logps/chosen": -1.3089931011199951,
+      "logps/rejected": -1.4399337768554688,
+      "loss": 1.65,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.3089931011199951,
+      "rewards/margins": 0.13094060122966766,
+      "rewards/rejected": -1.4399337768554688,
+      "step": 4095
+    },
+    {
+      "epoch": 2.194346880749289,
+      "grad_norm": 9.726428326876832,
+      "learning_rate": 2.0386916434539343e-07,
+      "logits/chosen": -0.16599038243293762,
+      "logits/rejected": -0.06162125989794731,
+      "logps/chosen": -1.1586806774139404,
+      "logps/rejected": -1.4184343814849854,
+      "loss": 1.5374,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.1586806774139404,
+      "rewards/margins": 0.2597537636756897,
+      "rewards/rejected": -1.4184343814849854,
+      "step": 4100
+    },
+    {
+      "epoch": 2.1970229135306907,
+      "grad_norm": 9.762474966817123,
+      "learning_rate": 2.0261573242594627e-07,
+      "logits/chosen": -0.18855324387550354,
+      "logits/rejected": -0.014698073267936707,
+      "logps/chosen": -1.280236840248108,
+      "logps/rejected": -1.449611783027649,
+      "loss": 1.6209,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.280236840248108,
+      "rewards/margins": 0.16937494277954102,
+      "rewards/rejected": -1.449611783027649,
+      "step": 4105
+    },
+    {
+      "epoch": 2.1996989463120924,
+      "grad_norm": 9.559046606243188,
+      "learning_rate": 2.0136518572996724e-07,
+      "logits/chosen": -0.14109668135643005,
+      "logits/rejected": -0.002549926284700632,
+      "logps/chosen": -1.2072608470916748,
+      "logps/rejected": -1.392521619796753,
+      "loss": 1.5823,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.2072608470916748,
+      "rewards/margins": 0.18526069819927216,
+      "rewards/rejected": -1.392521619796753,
+      "step": 4110
+    },
+    {
+      "epoch": 2.202374979093494,
+      "grad_norm": 7.579963806366611,
+      "learning_rate": 2.0011753639026617e-07,
+      "logits/chosen": -0.1384604573249817,
+      "logits/rejected": -0.1108267530798912,
+      "logps/chosen": -1.216487169265747,
+      "logps/rejected": -1.496044397354126,
+      "loss": 1.554,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.216487169265747,
+      "rewards/margins": 0.27955740690231323,
+      "rewards/rejected": -1.496044397354126,
+      "step": 4115
+    },
+    {
+      "epoch": 2.2050510118748954,
+      "grad_norm": 5.4827811338445,
+      "learning_rate": 1.988727965115421e-07,
+      "logits/chosen": -0.17637844383716583,
+      "logits/rejected": -0.12951934337615967,
+      "logps/chosen": -1.1752429008483887,
+      "logps/rejected": -1.428956151008606,
+      "loss": 1.5207,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.1752429008483887,
+      "rewards/margins": 0.2537131905555725,
+      "rewards/rejected": -1.428956151008606,
+      "step": 4120
+    },
+    {
+      "epoch": 2.207727044656297,
+      "grad_norm": 6.844973266135418,
+      "learning_rate": 1.9763097817026713e-07,
+      "logits/chosen": -0.212055966258049,
+      "logits/rejected": -0.04291188344359398,
+      "logps/chosen": -1.2000548839569092,
+      "logps/rejected": -1.415569543838501,
+      "loss": 1.5606,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.2000548839569092,
+      "rewards/margins": 0.21551451086997986,
+      "rewards/rejected": -1.415569543838501,
+      "step": 4125
+    },
+    {
+      "epoch": 2.210403077437699,
+      "grad_norm": 7.6682117055039125,
+      "learning_rate": 1.9639209341456796e-07,
+      "logits/chosen": -0.15989407896995544,
+      "logits/rejected": -0.09153582155704498,
+      "logps/chosen": -1.2050529718399048,
+      "logps/rejected": -1.4642305374145508,
+      "loss": 1.5452,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.2050529718399048,
+      "rewards/margins": 0.25917768478393555,
+      "rewards/rejected": -1.4642305374145508,
+      "step": 4130
+    },
+    {
+      "epoch": 2.2130791102191,
+      "grad_norm": 14.738132300531646,
+      "learning_rate": 1.951561542641102e-07,
+      "logits/chosen": -0.16685988008975983,
+      "logits/rejected": -0.1748218834400177,
+      "logps/chosen": -1.2561938762664795,
+      "logps/rejected": -1.4929753541946411,
+      "loss": 1.6073,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.2561938762664795,
+      "rewards/margins": 0.23678138852119446,
+      "rewards/rejected": -1.4929753541946411,
+      "step": 4135
+    },
+    {
+      "epoch": 2.215755143000502,
+      "grad_norm": 8.486783587459678,
+      "learning_rate": 1.939231727099806e-07,
+      "logits/chosen": -0.28792333602905273,
+      "logits/rejected": -0.21578791737556458,
+      "logps/chosen": -1.1780260801315308,
+      "logps/rejected": -1.4389264583587646,
+      "loss": 1.5305,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.1780260801315308,
+      "rewards/margins": 0.26090025901794434,
+      "rewards/rejected": -1.4389264583587646,
+      "step": 4140
+    },
+    {
+      "epoch": 2.2184311757819035,
+      "grad_norm": 8.16180910963854,
+      "learning_rate": 1.926931607145719e-07,
+      "logits/chosen": -0.09905657172203064,
+      "logits/rejected": 0.023060614243149757,
+      "logps/chosen": -1.2769787311553955,
+      "logps/rejected": -1.5458099842071533,
+      "loss": 1.6054,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.2769787311553955,
+      "rewards/margins": 0.26883116364479065,
+      "rewards/rejected": -1.5458099842071533,
+      "step": 4145
+    },
+    {
+      "epoch": 2.221107208563305,
+      "grad_norm": 8.070106294047847,
+      "learning_rate": 1.9146613021146564e-07,
+      "logits/chosen": -0.14280208945274353,
+      "logits/rejected": -0.0710291638970375,
+      "logps/chosen": -1.1692044734954834,
+      "logps/rejected": -1.434899926185608,
+      "loss": 1.5355,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.1692044734954834,
+      "rewards/margins": 0.26569539308547974,
+      "rewards/rejected": -1.434899926185608,
+      "step": 4150
+    },
+    {
+      "epoch": 2.2237832413447065,
+      "grad_norm": 8.036551483049287,
+      "learning_rate": 1.9024209310531736e-07,
+      "logits/chosen": -0.09412866830825806,
+      "logits/rejected": -0.12163051217794418,
+      "logps/chosen": -1.2059614658355713,
+      "logps/rejected": -1.4259283542633057,
+      "loss": 1.5517,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.2059614658355713,
+      "rewards/margins": 0.2199670821428299,
+      "rewards/rejected": -1.4259283542633057,
+      "step": 4155
+    },
+    {
+      "epoch": 2.2264592741261082,
+      "grad_norm": 9.278979880678373,
+      "learning_rate": 1.890210612717401e-07,
+      "logits/chosen": -0.17106539011001587,
+      "logits/rejected": -0.04072389751672745,
+      "logps/chosen": -1.2591288089752197,
+      "logps/rejected": -1.4694256782531738,
+      "loss": 1.6059,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.2591288089752197,
+      "rewards/margins": 0.21029677987098694,
+      "rewards/rejected": -1.4694256782531738,
+      "step": 4160
+    },
+    {
+      "epoch": 2.2291353069075095,
+      "grad_norm": 7.100599192462308,
+      "learning_rate": 1.8780304655719054e-07,
+      "logits/chosen": -0.1789466142654419,
+      "logits/rejected": -0.09607045352458954,
+      "logps/chosen": -1.26167631149292,
+      "logps/rejected": -1.4886329174041748,
+      "loss": 1.6086,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.26167631149292,
+      "rewards/margins": 0.22695684432983398,
+      "rewards/rejected": -1.4886329174041748,
+      "step": 4165
+    },
+    {
+      "epoch": 2.231811339688911,
+      "grad_norm": 8.472961986461971,
+      "learning_rate": 1.865880607788523e-07,
+      "logits/chosen": -0.0813198909163475,
+      "logits/rejected": -0.036982350051403046,
+      "logps/chosen": -1.2395800352096558,
+      "logps/rejected": -1.4827731847763062,
+      "loss": 1.575,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.2395800352096558,
+      "rewards/margins": 0.2431931495666504,
+      "rewards/rejected": -1.4827731847763062,
+      "step": 4170
+    },
+    {
+      "epoch": 2.234487372470313,
+      "grad_norm": 7.059742378344645,
+      "learning_rate": 1.8537611572452316e-07,
+      "logits/chosen": -0.1719505935907364,
+      "logits/rejected": -0.08126050978899002,
+      "logps/chosen": -1.2195240259170532,
+      "logps/rejected": -1.388882040977478,
+      "loss": 1.5901,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.2195240259170532,
+      "rewards/margins": 0.1693582832813263,
+      "rewards/rejected": -1.388882040977478,
+      "step": 4175
+    },
+    {
+      "epoch": 2.237163405251714,
+      "grad_norm": 10.458292911193098,
+      "learning_rate": 1.84167223152499e-07,
+      "logits/chosen": -0.20625898241996765,
+      "logits/rejected": -0.017097296193242073,
+      "logps/chosen": -1.1701924800872803,
+      "logps/rejected": -1.4478870630264282,
+      "loss": 1.5156,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.1701924800872803,
+      "rewards/margins": 0.2776945233345032,
+      "rewards/rejected": -1.4478870630264282,
+      "step": 4180
+    },
+    {
+      "epoch": 2.239839438033116,
+      "grad_norm": 8.809485044183429,
+      "learning_rate": 1.8296139479146112e-07,
+      "logits/chosen": -0.21345221996307373,
+      "logits/rejected": -0.21348659694194794,
+      "logps/chosen": -1.1450181007385254,
+      "logps/rejected": -1.4158300161361694,
+      "loss": 1.5114,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.1450181007385254,
+      "rewards/margins": 0.270812064409256,
+      "rewards/rejected": -1.4158300161361694,
+      "step": 4185
+    },
+    {
+      "epoch": 2.2425154708145176,
+      "grad_norm": 8.494491689490282,
+      "learning_rate": 1.8175864234036132e-07,
+      "logits/chosen": -0.10274024307727814,
+      "logits/rejected": -0.007196763064712286,
+      "logps/chosen": -1.2229564189910889,
+      "logps/rejected": -1.435478925704956,
+      "loss": 1.5785,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.2229564189910889,
+      "rewards/margins": 0.21252259612083435,
+      "rewards/rejected": -1.435478925704956,
+      "step": 4190
+    },
+    {
+      "epoch": 2.245191503595919,
+      "grad_norm": 5.3920501659104145,
+      "learning_rate": 1.805589774683094e-07,
+      "logits/chosen": -0.2902788817882538,
+      "logits/rejected": -0.15456892549991608,
+      "logps/chosen": -1.2663679122924805,
+      "logps/rejected": -1.4145959615707397,
+      "loss": 1.6248,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.2663679122924805,
+      "rewards/margins": 0.14822818338871002,
+      "rewards/rejected": -1.4145959615707397,
+      "step": 4195
+    },
+    {
+      "epoch": 2.2478675363773206,
+      "grad_norm": 8.537807124499022,
+      "learning_rate": 1.79362411814459e-07,
+      "logits/chosen": -0.0195014588534832,
+      "logits/rejected": -0.04255833104252815,
+      "logps/chosen": -1.2723864316940308,
+      "logps/rejected": -1.5024255514144897,
+      "loss": 1.601,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.2723864316940308,
+      "rewards/margins": 0.2300390899181366,
+      "rewards/rejected": -1.5024255514144897,
+      "step": 4200
+    },
+    {
+      "epoch": 2.2505435691587223,
+      "grad_norm": 8.356553445152443,
+      "learning_rate": 1.7816895698789552e-07,
+      "logits/chosen": -0.21217474341392517,
+      "logits/rejected": -0.14514055848121643,
+      "logps/chosen": -1.2000718116760254,
+      "logps/rejected": -1.4663290977478027,
+      "loss": 1.5215,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.2000718116760254,
+      "rewards/margins": 0.266257107257843,
+      "rewards/rejected": -1.4663290977478027,
+      "step": 4205
+    },
+    {
+      "epoch": 2.2532196019401236,
+      "grad_norm": 8.480499781545147,
+      "learning_rate": 1.7697862456752271e-07,
+      "logits/chosen": -0.19726264476776123,
+      "logits/rejected": -0.11173512786626816,
+      "logps/chosen": -1.213542103767395,
+      "logps/rejected": -1.5523083209991455,
+      "loss": 1.5522,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.213542103767395,
+      "rewards/margins": 0.3387662172317505,
+      "rewards/rejected": -1.5523083209991455,
+      "step": 4210
+    },
+    {
+      "epoch": 2.2558956347215253,
+      "grad_norm": 6.8471923153577725,
+      "learning_rate": 1.7579142610195124e-07,
+      "logits/chosen": -0.17487090826034546,
+      "logits/rejected": -0.06448483467102051,
+      "logps/chosen": -1.2431567907333374,
+      "logps/rejected": -1.452376365661621,
+      "loss": 1.5932,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.2431567907333374,
+      "rewards/margins": 0.2092195749282837,
+      "rewards/rejected": -1.452376365661621,
+      "step": 4215
+    },
+    {
+      "epoch": 2.258571667502927,
+      "grad_norm": 7.816383861597878,
+      "learning_rate": 1.7460737310938568e-07,
+      "logits/chosen": -0.22700047492980957,
+      "logits/rejected": -0.05646955966949463,
+      "logps/chosen": -1.2269337177276611,
+      "logps/rejected": -1.4205477237701416,
+      "loss": 1.5907,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.2269337177276611,
+      "rewards/margins": 0.19361400604248047,
+      "rewards/rejected": -1.4205477237701416,
+      "step": 4220
+    },
+    {
+      "epoch": 2.2612477002843283,
+      "grad_norm": 10.822570695434614,
+      "learning_rate": 1.734264770775133e-07,
+      "logits/chosen": -0.22074365615844727,
+      "logits/rejected": -0.07267844676971436,
+      "logps/chosen": -1.2606627941131592,
+      "logps/rejected": -1.5198907852172852,
+      "loss": 1.5868,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.2606627941131592,
+      "rewards/margins": 0.25922802090644836,
+      "rewards/rejected": -1.5198907852172852,
+      "step": 4225
+    },
+    {
+      "epoch": 2.26392373306573,
+      "grad_norm": 7.833504416013592,
+      "learning_rate": 1.7224874946339241e-07,
+      "logits/chosen": -0.2175557166337967,
+      "logits/rejected": -0.15451601147651672,
+      "logps/chosen": -1.2821791172027588,
+      "logps/rejected": -1.581095576286316,
+      "loss": 1.6205,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.2821791172027588,
+      "rewards/margins": 0.2989165782928467,
+      "rewards/rejected": -1.581095576286316,
+      "step": 4230
+    },
+    {
+      "epoch": 2.2665997658471317,
+      "grad_norm": 7.657476050617335,
+      "learning_rate": 1.7107420169334186e-07,
+      "logits/chosen": -0.14881347119808197,
+      "logits/rejected": -0.04365311190485954,
+      "logps/chosen": -1.2934807538986206,
+      "logps/rejected": -1.379242181777954,
+      "loss": 1.6734,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -1.2934807538986206,
+      "rewards/margins": 0.08576139062643051,
+      "rewards/rejected": -1.379242181777954,
+      "step": 4235
+    },
+    {
+      "epoch": 2.269275798628533,
+      "grad_norm": 8.447673546250616,
+      "learning_rate": 1.6990284516282893e-07,
+      "logits/chosen": -0.16056771576404572,
+      "logits/rejected": -0.06181789189577103,
+      "logps/chosen": -1.2400410175323486,
+      "logps/rejected": -1.3618853092193604,
+      "loss": 1.6195,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.2400410175323486,
+      "rewards/margins": 0.12184418737888336,
+      "rewards/rejected": -1.3618853092193604,
+      "step": 4240
+    },
+    {
+      "epoch": 2.2719518314099347,
+      "grad_norm": 7.006714687663983,
+      "learning_rate": 1.687346912363602e-07,
+      "logits/chosen": -0.23508353531360626,
+      "logits/rejected": -0.13327956199645996,
+      "logps/chosen": -1.2458765506744385,
+      "logps/rejected": -1.371870517730713,
+      "loss": 1.6025,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.2458765506744385,
+      "rewards/margins": 0.125994011759758,
+      "rewards/rejected": -1.371870517730713,
+      "step": 4245
+    },
+    {
+      "epoch": 2.2746278641913364,
+      "grad_norm": 5.102888582064418,
+      "learning_rate": 1.675697512473697e-07,
+      "logits/chosen": -0.1529920995235443,
+      "logits/rejected": -0.008715217001736164,
+      "logps/chosen": -1.2363736629486084,
+      "logps/rejected": -1.4021012783050537,
+      "loss": 1.6104,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.2363736629486084,
+      "rewards/margins": 0.16572771966457367,
+      "rewards/rejected": -1.4021012783050537,
+      "step": 4250
+    },
+    {
+      "epoch": 2.2773038969727377,
+      "grad_norm": 10.903502119428019,
+      "learning_rate": 1.6640803649811087e-07,
+      "logits/chosen": -0.21960356831550598,
+      "logits/rejected": -0.020394813269376755,
+      "logps/chosen": -1.2543021440505981,
+      "logps/rejected": -1.4993212223052979,
+      "loss": 1.5882,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.2543021440505981,
+      "rewards/margins": 0.24501904845237732,
+      "rewards/rejected": -1.4993212223052979,
+      "step": 4255
+    },
+    {
+      "epoch": 2.2799799297541394,
+      "grad_norm": 8.237759215848795,
+      "learning_rate": 1.6524955825954472e-07,
+      "logits/chosen": -0.18711432814598083,
+      "logits/rejected": -0.08437317609786987,
+      "logps/chosen": -1.1727478504180908,
+      "logps/rejected": -1.4643090963363647,
+      "loss": 1.5211,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.1727478504180908,
+      "rewards/margins": 0.29156142473220825,
+      "rewards/rejected": -1.4643090963363647,
+      "step": 4260
+    },
+    {
+      "epoch": 2.282655962535541,
+      "grad_norm": 6.595624346666196,
+      "learning_rate": 1.6409432777123277e-07,
+      "logits/chosen": -0.2630254626274109,
+      "logits/rejected": -0.12001305818557739,
+      "logps/chosen": -1.2227981090545654,
+      "logps/rejected": -1.4482836723327637,
+      "loss": 1.5619,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.2227981090545654,
+      "rewards/margins": 0.22548551857471466,
+      "rewards/rejected": -1.4482836723327637,
+      "step": 4265
+    },
+    {
+      "epoch": 2.285331995316943,
+      "grad_norm": 5.90018653535347,
+      "learning_rate": 1.6294235624122577e-07,
+      "logits/chosen": -0.10724867880344391,
+      "logits/rejected": 0.10244635492563248,
+      "logps/chosen": -1.2431776523590088,
+      "logps/rejected": -1.5928304195404053,
+      "loss": 1.5783,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.2431776523590088,
+      "rewards/margins": 0.34965258836746216,
+      "rewards/rejected": -1.5928304195404053,
+      "step": 4270
+    },
+    {
+      "epoch": 2.288008028098344,
+      "grad_norm": 8.869573524282167,
+      "learning_rate": 1.6179365484595697e-07,
+      "logits/chosen": -0.1616896092891693,
+      "logits/rejected": -0.09324827045202255,
+      "logps/chosen": -1.2973737716674805,
+      "logps/rejected": -1.4981673955917358,
+      "loss": 1.6484,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.2973737716674805,
+      "rewards/margins": 0.2007937729358673,
+      "rewards/rejected": -1.4981673955917358,
+      "step": 4275
+    },
+    {
+      "epoch": 2.290684060879746,
+      "grad_norm": 6.99952481826747,
+      "learning_rate": 1.60648234730132e-07,
+      "logits/chosen": -0.18767237663269043,
+      "logits/rejected": -0.1236768588423729,
+      "logps/chosen": -1.1969908475875854,
+      "logps/rejected": -1.3880600929260254,
+      "loss": 1.5465,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.1969908475875854,
+      "rewards/margins": 0.19106918573379517,
+      "rewards/rejected": -1.3880600929260254,
+      "step": 4280
+    },
+    {
+      "epoch": 2.293360093661147,
+      "grad_norm": 10.175508917314751,
+      "learning_rate": 1.595061070066222e-07,
+      "logits/chosen": -0.1178465262055397,
+      "logits/rejected": -0.1252368986606598,
+      "logps/chosen": -1.1660603284835815,
+      "logps/rejected": -1.4133269786834717,
+      "loss": 1.5046,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.1660603284835815,
+      "rewards/margins": 0.24726668000221252,
+      "rewards/rejected": -1.4133269786834717,
+      "step": 4285
+    },
+    {
+      "epoch": 2.296036126442549,
+      "grad_norm": 10.362950866375071,
+      "learning_rate": 1.5836728275635542e-07,
+      "logits/chosen": -0.23263172805309296,
+      "logits/rejected": -0.07924433052539825,
+      "logps/chosen": -1.269500970840454,
+      "logps/rejected": -1.436907172203064,
+      "loss": 1.611,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.269500970840454,
+      "rewards/margins": 0.16740620136260986,
+      "rewards/rejected": -1.436907172203064,
+      "step": 4290
+    },
+    {
+      "epoch": 2.2987121592239506,
+      "grad_norm": 8.875971527758225,
+      "learning_rate": 1.5723177302820984e-07,
+      "logits/chosen": -0.18567875027656555,
+      "logits/rejected": -0.1478704959154129,
+      "logps/chosen": -1.2316734790802002,
+      "logps/rejected": -1.4582233428955078,
+      "loss": 1.5816,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.2316734790802002,
+      "rewards/margins": 0.2265499383211136,
+      "rewards/rejected": -1.4582233428955078,
+      "step": 4295
+    },
+    {
+      "epoch": 2.3013881920053523,
+      "grad_norm": 11.997345478520547,
+      "learning_rate": 1.5609958883890544e-07,
+      "logits/chosen": -0.19638779759407043,
+      "logits/rejected": -0.1040961742401123,
+      "logps/chosen": -1.2302956581115723,
+      "logps/rejected": -1.3700530529022217,
+      "loss": 1.5969,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.2302956581115723,
+      "rewards/margins": 0.13975724577903748,
+      "rewards/rejected": -1.3700530529022217,
+      "step": 4300
+    },
+    {
+      "epoch": 2.3040642247867535,
+      "grad_norm": 9.273363482162067,
+      "learning_rate": 1.5497074117289865e-07,
+      "logits/chosen": -0.2623113989830017,
+      "logits/rejected": -0.16000813245773315,
+      "logps/chosen": -1.2087552547454834,
+      "logps/rejected": -1.4842298030853271,
+      "loss": 1.5421,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.2087552547454834,
+      "rewards/margins": 0.27547457814216614,
+      "rewards/rejected": -1.4842298030853271,
+      "step": 4305
+    },
+    {
+      "epoch": 2.3067402575681553,
+      "grad_norm": 8.284717897141489,
+      "learning_rate": 1.5384524098227402e-07,
+      "logits/chosen": -0.20485445857048035,
+      "logits/rejected": -0.035859428346157074,
+      "logps/chosen": -1.22329843044281,
+      "logps/rejected": -1.4752705097198486,
+      "loss": 1.5734,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.22329843044281,
+      "rewards/margins": 0.2519720196723938,
+      "rewards/rejected": -1.4752705097198486,
+      "step": 4310
+    },
+    {
+      "epoch": 2.3094162903495565,
+      "grad_norm": 7.149524801004495,
+      "learning_rate": 1.5272309918663974e-07,
+      "logits/chosen": -0.18672919273376465,
+      "logits/rejected": -0.06092787906527519,
+      "logps/chosen": -1.2216147184371948,
+      "logps/rejected": -1.401799201965332,
+      "loss": 1.5917,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.2216147184371948,
+      "rewards/margins": 0.1801842749118805,
+      "rewards/rejected": -1.401799201965332,
+      "step": 4315
+    },
+    {
+      "epoch": 2.3120923231309582,
+      "grad_norm": 8.360417135910502,
+      "learning_rate": 1.516043266730201e-07,
+      "logits/chosen": -0.20249688625335693,
+      "logits/rejected": -0.09940443933010101,
+      "logps/chosen": -1.2791379690170288,
+      "logps/rejected": -1.4019443988800049,
+      "loss": 1.6402,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.2791379690170288,
+      "rewards/margins": 0.1228063553571701,
+      "rewards/rejected": -1.4019443988800049,
+      "step": 4320
+    },
+    {
+      "epoch": 2.31476835591236,
+      "grad_norm": 11.234185197249264,
+      "learning_rate": 1.504889342957512e-07,
+      "logits/chosen": -0.21147003769874573,
+      "logits/rejected": -0.07601438462734222,
+      "logps/chosen": -1.203464150428772,
+      "logps/rejected": -1.4852374792099,
+      "loss": 1.5519,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.203464150428772,
+      "rewards/margins": 0.28177323937416077,
+      "rewards/rejected": -1.4852374792099,
+      "step": 4325
+    },
+    {
+      "epoch": 2.3174443886937617,
+      "grad_norm": 6.501783311715216,
+      "learning_rate": 1.4937693287637453e-07,
+      "logits/chosen": -0.1648644059896469,
+      "logits/rejected": -0.07460461556911469,
+      "logps/chosen": -1.304404377937317,
+      "logps/rejected": -1.4362707138061523,
+      "loss": 1.6622,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.304404377937317,
+      "rewards/margins": 0.131866455078125,
+      "rewards/rejected": -1.4362707138061523,
+      "step": 4330
+    },
+    {
+      "epoch": 2.320120421475163,
+      "grad_norm": 10.205396624845779,
+      "learning_rate": 1.4826833320353305e-07,
+      "logits/chosen": -0.14972421526908875,
+      "logits/rejected": -0.09903161227703094,
+      "logps/chosen": -1.281598448753357,
+      "logps/rejected": -1.4960078001022339,
+      "loss": 1.6442,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.281598448753357,
+      "rewards/margins": 0.2144092321395874,
+      "rewards/rejected": -1.4960078001022339,
+      "step": 4335
+    },
+    {
+      "epoch": 2.3227964542565647,
+      "grad_norm": 8.916306505575866,
+      "learning_rate": 1.4716314603286528e-07,
+      "logits/chosen": -0.18709716200828552,
+      "logits/rejected": -0.04617304354906082,
+      "logps/chosen": -1.1499556303024292,
+      "logps/rejected": -1.5380523204803467,
+      "loss": 1.4906,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.1499556303024292,
+      "rewards/margins": 0.3880966603755951,
+      "rewards/rejected": -1.5380523204803467,
+      "step": 4340
+    },
+    {
+      "epoch": 2.3254724870379664,
+      "grad_norm": 9.022344805920719,
+      "learning_rate": 1.4606138208690233e-07,
+      "logits/chosen": -0.2786336839199066,
+      "logits/rejected": -0.20892539620399475,
+      "logps/chosen": -1.3439069986343384,
+      "logps/rejected": -1.3802927732467651,
+      "loss": 1.7034,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.3439069986343384,
+      "rewards/margins": 0.03638586029410362,
+      "rewards/rejected": -1.3802927732467651,
+      "step": 4345
+    },
+    {
+      "epoch": 2.3281485198193677,
+      "grad_norm": 7.711159128746238,
+      "learning_rate": 1.4496305205496251e-07,
+      "logits/chosen": -0.1719704121351242,
+      "logits/rejected": -0.11904128640890121,
+      "logps/chosen": -1.239412784576416,
+      "logps/rejected": -1.5017035007476807,
+      "loss": 1.5726,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.239412784576416,
+      "rewards/margins": 0.2622907757759094,
+      "rewards/rejected": -1.5017035007476807,
+      "step": 4350
+    },
+    {
+      "epoch": 2.3308245526007694,
+      "grad_norm": 6.595624727606789,
+      "learning_rate": 1.4386816659304895e-07,
+      "logits/chosen": -0.2478518784046173,
+      "logits/rejected": -0.17337559163570404,
+      "logps/chosen": -1.2752437591552734,
+      "logps/rejected": -1.4763338565826416,
+      "loss": 1.6006,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.2752437591552734,
+      "rewards/margins": 0.2010900229215622,
+      "rewards/rejected": -1.4763338565826416,
+      "step": 4355
+    },
+    {
+      "epoch": 2.333500585382171,
+      "grad_norm": 7.420736127491981,
+      "learning_rate": 1.4277673632374492e-07,
+      "logits/chosen": -0.2834548354148865,
+      "logits/rejected": -0.10949674993753433,
+      "logps/chosen": -1.3200583457946777,
+      "logps/rejected": -1.4551689624786377,
+      "loss": 1.6533,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.3200583457946777,
+      "rewards/margins": 0.13511063158512115,
+      "rewards/rejected": -1.4551689624786377,
+      "step": 4360
+    },
+    {
+      "epoch": 2.3361766181635724,
+      "grad_norm": 7.875809964585136,
+      "learning_rate": 1.416887718361119e-07,
+      "logits/chosen": -0.13666775822639465,
+      "logits/rejected": -0.14465972781181335,
+      "logps/chosen": -1.1882762908935547,
+      "logps/rejected": -1.4436252117156982,
+      "loss": 1.5433,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.1882762908935547,
+      "rewards/margins": 0.2553488314151764,
+      "rewards/rejected": -1.4436252117156982,
+      "step": 4365
+    },
+    {
+      "epoch": 2.338852650944974,
+      "grad_norm": 9.979204525063246,
+      "learning_rate": 1.406042836855859e-07,
+      "logits/chosen": -0.2052156627178192,
+      "logits/rejected": -0.10023872554302216,
+      "logps/chosen": -1.155705451965332,
+      "logps/rejected": -1.534826636314392,
+      "loss": 1.4966,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.155705451965332,
+      "rewards/margins": 0.37912115454673767,
+      "rewards/rejected": -1.534826636314392,
+      "step": 4370
+    },
+    {
+      "epoch": 2.341528683726376,
+      "grad_norm": 10.66112545100067,
+      "learning_rate": 1.3952328239387595e-07,
+      "logits/chosen": -0.2963188588619232,
+      "logits/rejected": -0.14353522658348083,
+      "logps/chosen": -1.2621066570281982,
+      "logps/rejected": -1.4838224649429321,
+      "loss": 1.6012,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.2621066570281982,
+      "rewards/margins": 0.22171583771705627,
+      "rewards/rejected": -1.4838224649429321,
+      "step": 4375
+    },
+    {
+      "epoch": 2.344204716507777,
+      "grad_norm": 7.217437454739501,
+      "learning_rate": 1.3844577844886109e-07,
+      "logits/chosen": -0.20205311477184296,
+      "logits/rejected": -0.0537579171359539,
+      "logps/chosen": -1.2881264686584473,
+      "logps/rejected": -1.448127269744873,
+      "loss": 1.6281,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.2881264686584473,
+      "rewards/margins": 0.16000090539455414,
+      "rewards/rejected": -1.448127269744873,
+      "step": 4380
+    },
+    {
+      "epoch": 2.346880749289179,
+      "grad_norm": 9.649330496694866,
+      "learning_rate": 1.3737178230448955e-07,
+      "logits/chosen": -0.25397199392318726,
+      "logits/rejected": -0.12670288980007172,
+      "logps/chosen": -1.2086747884750366,
+      "logps/rejected": -1.5265108346939087,
+      "loss": 1.5239,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.2086747884750366,
+      "rewards/margins": 0.31783586740493774,
+      "rewards/rejected": -1.5265108346939087,
+      "step": 4385
+    },
+    {
+      "epoch": 2.3495567820705805,
+      "grad_norm": 6.571409844411033,
+      "learning_rate": 1.363013043806764e-07,
+      "logits/chosen": -0.19165340065956116,
+      "logits/rejected": -0.07862716913223267,
+      "logps/chosen": -1.2117689847946167,
+      "logps/rejected": -1.3977199792861938,
+      "loss": 1.5744,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.2117689847946167,
+      "rewards/margins": 0.18595094978809357,
+      "rewards/rejected": -1.3977199792861938,
+      "step": 4390
+    },
+    {
+      "epoch": 2.3522328148519818,
+      "grad_norm": 6.7949582778565665,
+      "learning_rate": 1.352343550632034e-07,
+      "logits/chosen": -0.1610317975282669,
+      "logits/rejected": -0.03247717395424843,
+      "logps/chosen": -1.237166166305542,
+      "logps/rejected": -1.5563886165618896,
+      "loss": 1.5657,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.237166166305542,
+      "rewards/margins": 0.3192223906517029,
+      "rewards/rejected": -1.5563886165618896,
+      "step": 4395
+    },
+    {
+      "epoch": 2.3549088476333835,
+      "grad_norm": 6.170379624826009,
+      "learning_rate": 1.3417094470361722e-07,
+      "logits/chosen": -0.2740820050239563,
+      "logits/rejected": -0.14671733975410461,
+      "logps/chosen": -1.198980450630188,
+      "logps/rejected": -1.4838148355484009,
+      "loss": 1.5309,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.198980450630188,
+      "rewards/margins": 0.28483447432518005,
+      "rewards/rejected": -1.4838148355484009,
+      "step": 4400
+    },
+    {
+      "epoch": 2.3549088476333835,
+      "eval_logits/chosen": 0.04385645315051079,
+      "eval_logits/rejected": 0.1132112517952919,
+      "eval_logps/chosen": -1.3050687313079834,
+      "eval_logps/rejected": -1.5041919946670532,
+      "eval_loss": 1.6466861963272095,
+      "eval_rewards/accuracies": 0.5727003216743469,
+      "eval_rewards/chosen": -1.3050687313079834,
+      "eval_rewards/margins": 0.1991230994462967,
+      "eval_rewards/rejected": -1.5041919946670532,
+      "eval_runtime": 40.8223,
+      "eval_samples_per_second": 32.948,
+      "eval_steps_per_second": 8.255,
+      "step": 4400
+    },
+    {
+      "epoch": 2.357584880414785,
+      "grad_norm": 7.5349630898521225,
+      "learning_rate": 1.3311108361913015e-07,
+      "logits/chosen": -0.2757181227207184,
+      "logits/rejected": -0.23484423756599426,
+      "logps/chosen": -1.213034987449646,
+      "logps/rejected": -1.3631339073181152,
+      "loss": 1.5892,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.213034987449646,
+      "rewards/margins": 0.1500987857580185,
+      "rewards/rejected": -1.3631339073181152,
+      "step": 4405
+    },
+    {
+      "epoch": 2.3602609131961865,
+      "grad_norm": 6.592700251523777,
+      "learning_rate": 1.3205478209251874e-07,
+      "logits/chosen": -0.19986172020435333,
+      "logits/rejected": -0.17671015858650208,
+      "logps/chosen": -1.3033101558685303,
+      "logps/rejected": -1.5694794654846191,
+      "loss": 1.6318,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.3033101558685303,
+      "rewards/margins": 0.2661692500114441,
+      "rewards/rejected": -1.5694794654846191,
+      "step": 4410
+    },
+    {
+      "epoch": 2.362936945977588,
+      "grad_norm": 7.564965929459961,
+      "learning_rate": 1.310020503720254e-07,
+      "logits/chosen": -0.16970126330852509,
+      "logits/rejected": -0.03718472644686699,
+      "logps/chosen": -1.2483030557632446,
+      "logps/rejected": -1.4484217166900635,
+      "loss": 1.5892,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.2483030557632446,
+      "rewards/margins": 0.20011882483959198,
+      "rewards/rejected": -1.4484217166900635,
+      "step": 4415
+    },
+    {
+      "epoch": 2.36561297875899,
+      "grad_norm": 8.4764314085826,
+      "learning_rate": 1.2995289867125752e-07,
+      "logits/chosen": -0.21752643585205078,
+      "logits/rejected": -0.15322408080101013,
+      "logps/chosen": -1.279433012008667,
+      "logps/rejected": -1.4012091159820557,
+      "loss": 1.6496,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.279433012008667,
+      "rewards/margins": 0.12177594006061554,
+      "rewards/rejected": -1.4012091159820557,
+      "step": 4420
+    },
+    {
+      "epoch": 2.368289011540391,
+      "grad_norm": 8.181139885606376,
+      "learning_rate": 1.2890733716908986e-07,
+      "logits/chosen": -0.20343129336833954,
+      "logits/rejected": -0.10383858531713486,
+      "logps/chosen": -1.1922613382339478,
+      "logps/rejected": -1.4458949565887451,
+      "loss": 1.5287,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.1922613382339478,
+      "rewards/margins": 0.2536335587501526,
+      "rewards/rejected": -1.4458949565887451,
+      "step": 4425
+    },
+    {
+      "epoch": 2.370965044321793,
+      "grad_norm": 5.960231929117382,
+      "learning_rate": 1.2786537600956454e-07,
+      "logits/chosen": -0.19824449717998505,
+      "logits/rejected": -0.05848120525479317,
+      "logps/chosen": -1.2846362590789795,
+      "logps/rejected": -1.5001429319381714,
+      "loss": 1.6152,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.2846362590789795,
+      "rewards/margins": 0.21550679206848145,
+      "rewards/rejected": -1.5001429319381714,
+      "step": 4430
+    },
+    {
+      "epoch": 2.3736410771031946,
+      "grad_norm": 6.663144772697994,
+      "learning_rate": 1.268270253017933e-07,
+      "logits/chosen": -0.28592783212661743,
+      "logits/rejected": -0.096594899892807,
+      "logps/chosen": -1.1786837577819824,
+      "logps/rejected": -1.4318368434906006,
+      "loss": 1.5258,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.1786837577819824,
+      "rewards/margins": 0.253152996301651,
+      "rewards/rejected": -1.4318368434906006,
+      "step": 4435
+    },
+    {
+      "epoch": 2.376317109884596,
+      "grad_norm": 9.6216050200051,
+      "learning_rate": 1.257922951198591e-07,
+      "logits/chosen": -0.2888815701007843,
+      "logits/rejected": -0.1167341023683548,
+      "logps/chosen": -1.2764893770217896,
+      "logps/rejected": -1.4089630842208862,
+      "loss": 1.6323,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.2764893770217896,
+      "rewards/margins": 0.1324736624956131,
+      "rewards/rejected": -1.4089630842208862,
+      "step": 4440
+    },
+    {
+      "epoch": 2.3789931426659976,
+      "grad_norm": 8.165315173346368,
+      "learning_rate": 1.24761195502719e-07,
+      "logits/chosen": -0.2527289092540741,
+      "logits/rejected": -0.10613594949245453,
+      "logps/chosen": -1.2289607524871826,
+      "logps/rejected": -1.4288841485977173,
+      "loss": 1.6052,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.2289607524871826,
+      "rewards/margins": 0.19992344081401825,
+      "rewards/rejected": -1.4288841485977173,
+      "step": 4445
+    },
+    {
+      "epoch": 2.3816691754473993,
+      "grad_norm": 9.785709031069278,
+      "learning_rate": 1.2373373645410573e-07,
+      "logits/chosen": -0.18451814353466034,
+      "logits/rejected": -0.07983697950839996,
+      "logps/chosen": -1.2855627536773682,
+      "logps/rejected": -1.500300407409668,
+      "loss": 1.6164,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.2855627536773682,
+      "rewards/margins": 0.2147376537322998,
+      "rewards/rejected": -1.500300407409668,
+      "step": 4450
+    },
+    {
+      "epoch": 2.384345208228801,
+      "grad_norm": 8.058393238708456,
+      "learning_rate": 1.2270992794243175e-07,
+      "logits/chosen": -0.26786744594573975,
+      "logits/rejected": -0.19314830005168915,
+      "logps/chosen": -1.2687526941299438,
+      "logps/rejected": -1.460627794265747,
+      "loss": 1.6077,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.2687526941299438,
+      "rewards/margins": 0.19187510013580322,
+      "rewards/rejected": -1.460627794265747,
+      "step": 4455
+    },
+    {
+      "epoch": 2.3870212410102023,
+      "grad_norm": 7.4465935675716235,
+      "learning_rate": 1.2168977990069147e-07,
+      "logits/chosen": -0.27446383237838745,
+      "logits/rejected": -0.0762770026922226,
+      "logps/chosen": -1.187028169631958,
+      "logps/rejected": -1.5143496990203857,
+      "loss": 1.5082,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.187028169631958,
+      "rewards/margins": 0.3273215591907501,
+      "rewards/rejected": -1.5143496990203857,
+      "step": 4460
+    },
+    {
+      "epoch": 2.389697273791604,
+      "grad_norm": 6.9486768670050285,
+      "learning_rate": 1.206733022263659e-07,
+      "logits/chosen": -0.27223438024520874,
+      "logits/rejected": -0.11611995846033096,
+      "logps/chosen": -1.287155270576477,
+      "logps/rejected": -1.4335745573043823,
+      "loss": 1.645,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.287155270576477,
+      "rewards/margins": 0.146419495344162,
+      "rewards/rejected": -1.4335745573043823,
+      "step": 4465
+    },
+    {
+      "epoch": 2.3923733065730053,
+      "grad_norm": 8.014042276809453,
+      "learning_rate": 1.1966050478132572e-07,
+      "logits/chosen": -0.1054086685180664,
+      "logits/rejected": -0.039462633430957794,
+      "logps/chosen": -1.1409763097763062,
+      "logps/rejected": -1.3532037734985352,
+      "loss": 1.5266,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.1409763097763062,
+      "rewards/margins": 0.21222741901874542,
+      "rewards/rejected": -1.3532037734985352,
+      "step": 4470
+    },
+    {
+      "epoch": 2.395049339354407,
+      "grad_norm": 8.033512264834458,
+      "learning_rate": 1.1865139739173635e-07,
+      "logits/chosen": -0.2219352424144745,
+      "logits/rejected": -0.0237773135304451,
+      "logps/chosen": -1.2053747177124023,
+      "logps/rejected": -1.429945707321167,
+      "loss": 1.5515,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.2053747177124023,
+      "rewards/margins": 0.22457103431224823,
+      "rewards/rejected": -1.429945707321167,
+      "step": 4475
+    },
+    {
+      "epoch": 2.3977253721358087,
+      "grad_norm": 8.917144958810724,
+      "learning_rate": 1.1764598984796187e-07,
+      "logits/chosen": -0.17329058051109314,
+      "logits/rejected": -0.10808472335338593,
+      "logps/chosen": -1.143885612487793,
+      "logps/rejected": -1.339099407196045,
+      "loss": 1.5354,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.143885612487793,
+      "rewards/margins": 0.19521372020244598,
+      "rewards/rejected": -1.339099407196045,
+      "step": 4480
+    },
+    {
+      "epoch": 2.4004014049172104,
+      "grad_norm": 11.834309102099507,
+      "learning_rate": 1.1664429190447095e-07,
+      "logits/chosen": -0.2013109028339386,
+      "logits/rejected": -0.1036062240600586,
+      "logps/chosen": -1.2618813514709473,
+      "logps/rejected": -1.518686056137085,
+      "loss": 1.6,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.2618813514709473,
+      "rewards/margins": 0.2568047046661377,
+      "rewards/rejected": -1.518686056137085,
+      "step": 4485
+    },
+    {
+      "epoch": 2.4030774376986117,
+      "grad_norm": 7.175929463146946,
+      "learning_rate": 1.1564631327974122e-07,
+      "logits/chosen": -0.2383660525083542,
+      "logits/rejected": -0.04933794587850571,
+      "logps/chosen": -1.2206124067306519,
+      "logps/rejected": -1.5133615732192993,
+      "loss": 1.5399,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.2206124067306519,
+      "rewards/margins": 0.2927490770816803,
+      "rewards/rejected": -1.5133615732192993,
+      "step": 4490
+    },
+    {
+      "epoch": 2.4057534704800134,
+      "grad_norm": 9.16204325508004,
+      "learning_rate": 1.1465206365616587e-07,
+      "logits/chosen": -0.2820783853530884,
+      "logits/rejected": -0.09989559650421143,
+      "logps/chosen": -1.2064526081085205,
+      "logps/rejected": -1.4397554397583008,
+      "loss": 1.5541,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.2064526081085205,
+      "rewards/margins": 0.23330290615558624,
+      "rewards/rejected": -1.4397554397583008,
+      "step": 4495
+    },
+    {
+      "epoch": 2.408429503261415,
+      "grad_norm": 6.78544051173504,
+      "learning_rate": 1.1366155267995887e-07,
+      "logits/chosen": -0.11139881610870361,
+      "logits/rejected": -0.11971314251422882,
+      "logps/chosen": -1.2364609241485596,
+      "logps/rejected": -1.4335353374481201,
+      "loss": 1.5937,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.2364609241485596,
+      "rewards/margins": 0.19707438349723816,
+      "rewards/rejected": -1.4335353374481201,
+      "step": 4500
+    },
+    {
+      "epoch": 2.4111055360428164,
+      "grad_norm": 10.310534197220848,
+      "learning_rate": 1.1267478996106228e-07,
+      "logits/chosen": -0.2633117437362671,
+      "logits/rejected": -0.14126761257648468,
+      "logps/chosen": -1.2213430404663086,
+      "logps/rejected": -1.5242010354995728,
+      "loss": 1.56,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.2213430404663086,
+      "rewards/margins": 0.3028578460216522,
+      "rewards/rejected": -1.5242010354995728,
+      "step": 4505
+    },
+    {
+      "epoch": 2.413781568824218,
+      "grad_norm": 11.1093160012141,
+      "learning_rate": 1.116917850730521e-07,
+      "logits/chosen": -0.24887220561504364,
+      "logits/rejected": -0.14276175200939178,
+      "logps/chosen": -1.1748237609863281,
+      "logps/rejected": -1.3930141925811768,
+      "loss": 1.5492,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.1748237609863281,
+      "rewards/margins": 0.21819040179252625,
+      "rewards/rejected": -1.3930141925811768,
+      "step": 4510
+    },
+    {
+      "epoch": 2.41645760160562,
+      "grad_norm": 6.210320759773411,
+      "learning_rate": 1.1071254755304637e-07,
+      "logits/chosen": -0.22284992039203644,
+      "logits/rejected": -0.16558507084846497,
+      "logps/chosen": -1.2325079441070557,
+      "logps/rejected": -1.4816372394561768,
+      "loss": 1.5629,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.2325079441070557,
+      "rewards/margins": 0.24912920594215393,
+      "rewards/rejected": -1.4816372394561768,
+      "step": 4515
+    },
+    {
+      "epoch": 2.419133634387021,
+      "grad_norm": 7.705672138397239,
+      "learning_rate": 1.0973708690161143e-07,
+      "logits/chosen": -0.20282363891601562,
+      "logits/rejected": -0.13200493156909943,
+      "logps/chosen": -1.209581732749939,
+      "logps/rejected": -1.4997475147247314,
+      "loss": 1.5331,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.209581732749939,
+      "rewards/margins": 0.2901657223701477,
+      "rewards/rejected": -1.4997475147247314,
+      "step": 4520
+    },
+    {
+      "epoch": 2.421809667168423,
+      "grad_norm": 10.387820564546052,
+      "learning_rate": 1.0876541258267119e-07,
+      "logits/chosen": -0.2831111550331116,
+      "logits/rejected": -0.13041360676288605,
+      "logps/chosen": -1.3232536315917969,
+      "logps/rejected": -1.5723609924316406,
+      "loss": 1.6341,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.3232536315917969,
+      "rewards/margins": 0.24910736083984375,
+      "rewards/rejected": -1.5723609924316406,
+      "step": 4525
+    },
+    {
+      "epoch": 2.4244856999498245,
+      "grad_norm": 6.50545879373684,
+      "learning_rate": 1.0779753402341379e-07,
+      "logits/chosen": -0.242244154214859,
+      "logits/rejected": -0.16366708278656006,
+      "logps/chosen": -1.2527737617492676,
+      "logps/rejected": -1.4390571117401123,
+      "loss": 1.6024,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.2527737617492676,
+      "rewards/margins": 0.18628332018852234,
+      "rewards/rejected": -1.4390571117401123,
+      "step": 4530
+    },
+    {
+      "epoch": 2.427161732731226,
+      "grad_norm": 7.166247614143461,
+      "learning_rate": 1.0683346061420157e-07,
+      "logits/chosen": -0.0786890909075737,
+      "logits/rejected": 0.01348264329135418,
+      "logps/chosen": -1.2025784254074097,
+      "logps/rejected": -1.4283499717712402,
+      "loss": 1.5804,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.2025784254074097,
+      "rewards/margins": 0.22577135264873505,
+      "rewards/rejected": -1.4283499717712402,
+      "step": 4535
+    },
+    {
+      "epoch": 2.4298377655126275,
+      "grad_norm": 7.152108536625329,
+      "learning_rate": 1.0587320170847874e-07,
+      "logits/chosen": -0.1070355772972107,
+      "logits/rejected": -0.047557950019836426,
+      "logps/chosen": -1.1764482259750366,
+      "logps/rejected": -1.3941954374313354,
+      "loss": 1.5467,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.1764482259750366,
+      "rewards/margins": 0.2177472561597824,
+      "rewards/rejected": -1.3941954374313354,
+      "step": 4540
+    },
+    {
+      "epoch": 2.4325137982940293,
+      "grad_norm": 7.238332196173785,
+      "learning_rate": 1.0491676662268156e-07,
+      "logits/chosen": -0.1882878541946411,
+      "logits/rejected": -0.07006659358739853,
+      "logps/chosen": -1.1832000017166138,
+      "logps/rejected": -1.424682378768921,
+      "loss": 1.5582,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.1832000017166138,
+      "rewards/margins": 0.24148237705230713,
+      "rewards/rejected": -1.424682378768921,
+      "step": 4545
+    },
+    {
+      "epoch": 2.4351898310754305,
+      "grad_norm": 10.485834168254481,
+      "learning_rate": 1.0396416463614732e-07,
+      "logits/chosen": -0.24194899201393127,
+      "logits/rejected": -0.14754289388656616,
+      "logps/chosen": -1.1658766269683838,
+      "logps/rejected": -1.4418859481811523,
+      "loss": 1.5452,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.1658766269683838,
+      "rewards/margins": 0.27600932121276855,
+      "rewards/rejected": -1.4418859481811523,
+      "step": 4550
+    },
+    {
+      "epoch": 2.4378658638568322,
+      "grad_norm": 7.914123241626193,
+      "learning_rate": 1.0301540499102479e-07,
+      "logits/chosen": -0.2195291817188263,
+      "logits/rejected": -0.1631467640399933,
+      "logps/chosen": -1.2823065519332886,
+      "logps/rejected": -1.430878758430481,
+      "loss": 1.6272,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.2823065519332886,
+      "rewards/margins": 0.14857208728790283,
+      "rewards/rejected": -1.430878758430481,
+      "step": 4555
+    },
+    {
+      "epoch": 2.440541896638234,
+      "grad_norm": 9.804187017833879,
+      "learning_rate": 1.0207049689218405e-07,
+      "logits/chosen": -0.2617951035499573,
+      "logits/rejected": -0.1206180602312088,
+      "logps/chosen": -1.1838595867156982,
+      "logps/rejected": -1.3840501308441162,
+      "loss": 1.5599,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.1838595867156982,
+      "rewards/margins": 0.20019057393074036,
+      "rewards/rejected": -1.3840501308441162,
+      "step": 4560
+    },
+    {
+      "epoch": 2.4432179294196352,
+      "grad_norm": 9.221087836615421,
+      "learning_rate": 1.0112944950712782e-07,
+      "logits/chosen": -0.1988077610731125,
+      "logits/rejected": -0.140293687582016,
+      "logps/chosen": -1.284519076347351,
+      "logps/rejected": -1.474821925163269,
+      "loss": 1.6212,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.284519076347351,
+      "rewards/margins": 0.19030292332172394,
+      "rewards/rejected": -1.474821925163269,
+      "step": 4565
+    },
+    {
+      "epoch": 2.445893962201037,
+      "grad_norm": 6.782121730236154,
+      "learning_rate": 1.0019227196590174e-07,
+      "logits/chosen": -0.13662488758563995,
+      "logits/rejected": -0.0032837330363690853,
+      "logps/chosen": -1.2362960577011108,
+      "logps/rejected": -1.4920551776885986,
+      "loss": 1.5736,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.2362960577011108,
+      "rewards/margins": 0.255759060382843,
+      "rewards/rejected": -1.4920551776885986,
+      "step": 4570
+    },
+    {
+      "epoch": 2.4485699949824387,
+      "grad_norm": 7.190362233091777,
+      "learning_rate": 9.925897336100664e-08,
+      "logits/chosen": -0.13475404679775238,
+      "logits/rejected": -0.09383302927017212,
+      "logps/chosen": -1.2316467761993408,
+      "logps/rejected": -1.4339395761489868,
+      "loss": 1.5886,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.2316467761993408,
+      "rewards/margins": 0.20229271054267883,
+      "rewards/rejected": -1.4339395761489868,
+      "step": 4575
+    },
+    {
+      "epoch": 2.45124602776384,
+      "grad_norm": 8.377076003160667,
+      "learning_rate": 9.832956274730946e-08,
+      "logits/chosen": -0.2030809372663498,
+      "logits/rejected": -0.1647244095802307,
+      "logps/chosen": -1.179179072380066,
+      "logps/rejected": -1.4651811122894287,
+      "loss": 1.5232,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.179179072380066,
+      "rewards/margins": 0.2860020101070404,
+      "rewards/rejected": -1.4651811122894287,
+      "step": 4580
+    },
+    {
+      "epoch": 2.4539220605452416,
+      "grad_norm": 6.6990802347524685,
+      "learning_rate": 9.740404914195633e-08,
+      "logits/chosen": -0.16281422972679138,
+      "logits/rejected": -0.04782953858375549,
+      "logps/chosen": -1.291107416152954,
+      "logps/rejected": -1.4581815004348755,
+      "loss": 1.6578,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.291107416152954,
+      "rewards/margins": 0.16707420349121094,
+      "rewards/rejected": -1.4581815004348755,
+      "step": 4585
+    },
+    {
+      "epoch": 2.4565980933266434,
+      "grad_norm": 6.900910408002965,
+      "learning_rate": 9.648244152428392e-08,
+      "logits/chosen": -0.27961355447769165,
+      "logits/rejected": -0.11388464272022247,
+      "logps/chosen": -1.1816189289093018,
+      "logps/rejected": -1.426766037940979,
+      "loss": 1.5509,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.1816189289093018,
+      "rewards/margins": 0.24514737725257874,
+      "rewards/rejected": -1.426766037940979,
+      "step": 4590
+    },
+    {
+      "epoch": 2.4592741261080446,
+      "grad_norm": 7.585177745899499,
+      "learning_rate": 9.556474883573379e-08,
+      "logits/chosen": -0.23860666155815125,
+      "logits/rejected": -0.13447125256061554,
+      "logps/chosen": -1.2374709844589233,
+      "logps/rejected": -1.509106993675232,
+      "loss": 1.5831,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.2374709844589233,
+      "rewards/margins": 0.271636039018631,
+      "rewards/rejected": -1.509106993675232,
+      "step": 4595
+    },
+    {
+      "epoch": 2.4619501588894463,
+      "grad_norm": 8.26242453249571,
+      "learning_rate": 9.465097997976412e-08,
+      "logits/chosen": -0.21761243045330048,
+      "logits/rejected": 0.014684316702187061,
+      "logps/chosen": -1.2814009189605713,
+      "logps/rejected": -1.5540639162063599,
+      "loss": 1.6085,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.2814009189605713,
+      "rewards/margins": 0.2726629972457886,
+      "rewards/rejected": -1.5540639162063599,
+      "step": 4600
+    },
+    {
+      "epoch": 2.464626191670848,
+      "grad_norm": 8.565484930594367,
+      "learning_rate": 9.374114382176457e-08,
+      "logits/chosen": -0.15770654380321503,
+      "logits/rejected": -0.048599086701869965,
+      "logps/chosen": -1.258568525314331,
+      "logps/rejected": -1.5116450786590576,
+      "loss": 1.6018,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.258568525314331,
+      "rewards/margins": 0.2530765235424042,
+      "rewards/rejected": -1.5116450786590576,
+      "step": 4605
+    },
+    {
+      "epoch": 2.46730222445225,
+      "grad_norm": 8.024702890810318,
+      "learning_rate": 9.283524918896945e-08,
+      "logits/chosen": -0.20571458339691162,
+      "logits/rejected": -0.06943124532699585,
+      "logps/chosen": -1.2410836219787598,
+      "logps/rejected": -1.5107513666152954,
+      "loss": 1.5583,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.2410836219787598,
+      "rewards/margins": 0.26966771483421326,
+      "rewards/rejected": -1.5107513666152954,
+      "step": 4610
+    },
+    {
+      "epoch": 2.469978257233651,
+      "grad_norm": 7.078771121377501,
+      "learning_rate": 9.193330487037232e-08,
+      "logits/chosen": -0.149086594581604,
+      "logits/rejected": -0.0335305891931057,
+      "logps/chosen": -1.2589304447174072,
+      "logps/rejected": -1.5451947450637817,
+      "loss": 1.5774,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.2589304447174072,
+      "rewards/margins": 0.28626421093940735,
+      "rewards/rejected": -1.5451947450637817,
+      "step": 4615
+    },
+    {
+      "epoch": 2.4726542900150528,
+      "grad_norm": 7.410281710233767,
+      "learning_rate": 9.103531961664118e-08,
+      "logits/chosen": -0.1585276871919632,
+      "logits/rejected": 0.013647395186126232,
+      "logps/chosen": -1.1524250507354736,
+      "logps/rejected": -1.4461370706558228,
+      "loss": 1.4992,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.1524250507354736,
+      "rewards/margins": 0.29371196031570435,
+      "rewards/rejected": -1.4461370706558228,
+      "step": 4620
+    },
+    {
+      "epoch": 2.475330322796454,
+      "grad_norm": 6.4861215775128365,
+      "learning_rate": 9.014130214003269e-08,
+      "logits/chosen": -0.27106332778930664,
+      "logits/rejected": -0.22946906089782715,
+      "logps/chosen": -1.2726995944976807,
+      "logps/rejected": -1.4708665609359741,
+      "loss": 1.5953,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.2726995944976807,
+      "rewards/margins": 0.19816681742668152,
+      "rewards/rejected": -1.4708665609359741,
+      "step": 4625
+    },
+    {
+      "epoch": 2.4780063555778558,
+      "grad_norm": 11.593695563467303,
+      "learning_rate": 8.925126111430848e-08,
+      "logits/chosen": -0.18593397736549377,
+      "logits/rejected": -0.11719465255737305,
+      "logps/chosen": -1.1971778869628906,
+      "logps/rejected": -1.4603909254074097,
+      "loss": 1.5598,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.1971778869628906,
+      "rewards/margins": 0.26321300864219666,
+      "rewards/rejected": -1.4603909254074097,
+      "step": 4630
+    },
+    {
+      "epoch": 2.4806823883592575,
+      "grad_norm": 9.842887811694869,
+      "learning_rate": 8.83652051746504e-08,
+      "logits/chosen": -0.09734443575143814,
+      "logits/rejected": 0.02725895121693611,
+      "logps/chosen": -1.2012536525726318,
+      "logps/rejected": -1.4989259243011475,
+      "loss": 1.5646,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.2012536525726318,
+      "rewards/margins": 0.2976721227169037,
+      "rewards/rejected": -1.4989259243011475,
+      "step": 4635
+    },
+    {
+      "epoch": 2.483358421140659,
+      "grad_norm": 7.375338188343925,
+      "learning_rate": 8.748314291757696e-08,
+      "logits/chosen": -0.14858081936836243,
+      "logits/rejected": -0.05358070880174637,
+      "logps/chosen": -1.2337394952774048,
+      "logps/rejected": -1.4716202020645142,
+      "loss": 1.5705,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.2337394952774048,
+      "rewards/margins": 0.2378806620836258,
+      "rewards/rejected": -1.4716202020645142,
+      "step": 4640
+    },
+    {
+      "epoch": 2.4860344539220605,
+      "grad_norm": 9.083724144875998,
+      "learning_rate": 8.660508290086032e-08,
+      "logits/chosen": -0.15422284603118896,
+      "logits/rejected": -0.032006729394197464,
+      "logps/chosen": -1.211015224456787,
+      "logps/rejected": -1.4706156253814697,
+      "loss": 1.5634,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.211015224456787,
+      "rewards/margins": 0.2596002519130707,
+      "rewards/rejected": -1.4706156253814697,
+      "step": 4645
+    },
+    {
+      "epoch": 2.488710486703462,
+      "grad_norm": 8.495728501232144,
+      "learning_rate": 8.573103364344231e-08,
+      "logits/chosen": -0.19348394870758057,
+      "logits/rejected": 0.028809908777475357,
+      "logps/chosen": -1.2423301935195923,
+      "logps/rejected": -1.5411301851272583,
+      "loss": 1.5551,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.2423301935195923,
+      "rewards/margins": 0.298799991607666,
+      "rewards/rejected": -1.5411301851272583,
+      "step": 4650
+    },
+    {
+      "epoch": 2.4913865194848634,
+      "grad_norm": 7.953930669088138,
+      "learning_rate": 8.486100362535292e-08,
+      "logits/chosen": -0.21532246470451355,
+      "logits/rejected": -0.08617198467254639,
+      "logps/chosen": -1.2144615650177002,
+      "logps/rejected": -1.3963156938552856,
+      "loss": 1.5693,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.2144615650177002,
+      "rewards/margins": 0.18185389041900635,
+      "rewards/rejected": -1.3963156938552856,
+      "step": 4655
+    },
+    {
+      "epoch": 2.494062552266265,
+      "grad_norm": 6.625260423362665,
+      "learning_rate": 8.399500128762693e-08,
+      "logits/chosen": -0.20581689476966858,
+      "logits/rejected": -0.09944789111614227,
+      "logps/chosen": -1.2891731262207031,
+      "logps/rejected": -1.4467524290084839,
+      "loss": 1.6299,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.2891731262207031,
+      "rewards/margins": 0.15757936239242554,
+      "rewards/rejected": -1.4467524290084839,
+      "step": 4660
+    },
+    {
+      "epoch": 2.496738585047667,
+      "grad_norm": 7.384548483424394,
+      "learning_rate": 8.313303503222313e-08,
+      "logits/chosen": -0.1977105438709259,
+      "logits/rejected": -0.14704328775405884,
+      "logps/chosen": -1.3137004375457764,
+      "logps/rejected": -1.5258251428604126,
+      "loss": 1.645,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.3137004375457764,
+      "rewards/margins": 0.21212443709373474,
+      "rewards/rejected": -1.5258251428604126,
+      "step": 4665
+    },
+    {
+      "epoch": 2.4994146178290686,
+      "grad_norm": 8.198769766484682,
+      "learning_rate": 8.227511322194164e-08,
+      "logits/chosen": -0.18010950088500977,
+      "logits/rejected": -0.06886539608240128,
+      "logps/chosen": -1.280510663986206,
+      "logps/rejected": -1.4507843255996704,
+      "loss": 1.6213,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.280510663986206,
+      "rewards/margins": 0.17027351260185242,
+      "rewards/rejected": -1.4507843255996704,
+      "step": 4670
+    },
+    {
+      "epoch": 2.50209065061047,
+      "grad_norm": 6.855115745421152,
+      "learning_rate": 8.142124418034385e-08,
+      "logits/chosen": -0.1507149189710617,
+      "logits/rejected": -0.026994511485099792,
+      "logps/chosen": -1.1871156692504883,
+      "logps/rejected": -1.4419450759887695,
+      "loss": 1.5485,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.1871156692504883,
+      "rewards/margins": 0.25482943654060364,
+      "rewards/rejected": -1.4419450759887695,
+      "step": 4675
+    },
+    {
+      "epoch": 2.5047666833918716,
+      "grad_norm": 8.982655143567454,
+      "learning_rate": 8.057143619167073e-08,
+      "logits/chosen": -0.14716149866580963,
+      "logits/rejected": -0.050116412341594696,
+      "logps/chosen": -1.233823537826538,
+      "logps/rejected": -1.4536529779434204,
+      "loss": 1.587,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.233823537826538,
+      "rewards/margins": 0.2198294699192047,
+      "rewards/rejected": -1.4536529779434204,
+      "step": 4680
+    },
+    {
+      "epoch": 2.507442716173273,
+      "grad_norm": 6.2140511328578905,
+      "learning_rate": 7.97256975007633e-08,
+      "logits/chosen": -0.2550865113735199,
+      "logits/rejected": -0.09868437051773071,
+      "logps/chosen": -1.2446149587631226,
+      "logps/rejected": -1.4486770629882812,
+      "loss": 1.5898,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.2446149587631226,
+      "rewards/margins": 0.20406225323677063,
+      "rewards/rejected": -1.4486770629882812,
+      "step": 4685
+    },
+    {
+      "epoch": 2.5101187489546746,
+      "grad_norm": 9.244400120324464,
+      "learning_rate": 7.888403631298186e-08,
+      "logits/chosen": -0.14784127473831177,
+      "logits/rejected": -0.11010942608118057,
+      "logps/chosen": -1.2629327774047852,
+      "logps/rejected": -1.443296194076538,
+      "loss": 1.6106,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.2629327774047852,
+      "rewards/margins": 0.18036355078220367,
+      "rewards/rejected": -1.443296194076538,
+      "step": 4690
+    },
+    {
+      "epoch": 2.5127947817360763,
+      "grad_norm": 8.161542062031891,
+      "learning_rate": 7.804646079412719e-08,
+      "logits/chosen": -0.1550903618335724,
+      "logits/rejected": 0.014736851677298546,
+      "logps/chosen": -1.2535508871078491,
+      "logps/rejected": -1.4470226764678955,
+      "loss": 1.5888,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.2535508871078491,
+      "rewards/margins": 0.19347189366817474,
+      "rewards/rejected": -1.4470226764678955,
+      "step": 4695
+    },
+    {
+      "epoch": 2.515470814517478,
+      "grad_norm": 7.057383355262316,
+      "learning_rate": 7.72129790703604e-08,
+      "logits/chosen": -0.2678453326225281,
+      "logits/rejected": -0.17103447020053864,
+      "logps/chosen": -1.2096585035324097,
+      "logps/rejected": -1.4120450019836426,
+      "loss": 1.5652,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.2096585035324097,
+      "rewards/margins": 0.20238666236400604,
+      "rewards/rejected": -1.4120450019836426,
+      "step": 4700
+    },
+    {
+      "epoch": 2.5181468472988793,
+      "grad_norm": 8.83446443379937,
+      "learning_rate": 7.638359922812504e-08,
+      "logits/chosen": -0.11662398278713226,
+      "logits/rejected": -0.09670352190732956,
+      "logps/chosen": -1.2910524606704712,
+      "logps/rejected": -1.4886689186096191,
+      "loss": 1.6368,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.2910524606704712,
+      "rewards/margins": 0.19761626422405243,
+      "rewards/rejected": -1.4886689186096191,
+      "step": 4705
+    },
+    {
+      "epoch": 2.520822880080281,
+      "grad_norm": 10.996392501015148,
+      "learning_rate": 7.555832931406774e-08,
+      "logits/chosen": -0.19205592572689056,
+      "logits/rejected": -0.06259692460298538,
+      "logps/chosen": -1.2577614784240723,
+      "logps/rejected": -1.4748903512954712,
+      "loss": 1.5913,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.2577614784240723,
+      "rewards/margins": 0.21712878346443176,
+      "rewards/rejected": -1.4748903512954712,
+      "step": 4710
+    },
+    {
+      "epoch": 2.5234989128616827,
+      "grad_norm": 7.080472540676264,
+      "learning_rate": 7.47371773349611e-08,
+      "logits/chosen": -0.2080480307340622,
+      "logits/rejected": -0.1923224925994873,
+      "logps/chosen": -1.3185064792633057,
+      "logps/rejected": -1.525776982307434,
+      "loss": 1.6493,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.3185064792633057,
+      "rewards/margins": 0.2072705775499344,
+      "rewards/rejected": -1.525776982307434,
+      "step": 4715
+    },
+    {
+      "epoch": 2.526174945643084,
+      "grad_norm": 9.847853778010714,
+      "learning_rate": 7.392015125762496e-08,
+      "logits/chosen": -0.14937856793403625,
+      "logits/rejected": -0.07092615962028503,
+      "logps/chosen": -1.1753498315811157,
+      "logps/rejected": -1.4301412105560303,
+      "loss": 1.5205,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.1753498315811157,
+      "rewards/margins": 0.2547914385795593,
+      "rewards/rejected": -1.4301412105560303,
+      "step": 4720
+    },
+    {
+      "epoch": 2.5288509784244857,
+      "grad_norm": 8.166106113851754,
+      "learning_rate": 7.310725900885018e-08,
+      "logits/chosen": -0.2096148282289505,
+      "logits/rejected": -0.1626475304365158,
+      "logps/chosen": -1.258299469947815,
+      "logps/rejected": -1.4184331893920898,
+      "loss": 1.6108,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.258299469947815,
+      "rewards/margins": 0.16013376414775848,
+      "rewards/rejected": -1.4184331893920898,
+      "step": 4725
+    },
+    {
+      "epoch": 2.5315270112058874,
+      "grad_norm": 8.605523645573196,
+      "learning_rate": 7.229850847532076e-08,
+      "logits/chosen": -0.17315617203712463,
+      "logits/rejected": -0.07473836839199066,
+      "logps/chosen": -1.1580817699432373,
+      "logps/rejected": -1.3972079753875732,
+      "loss": 1.5282,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.1580817699432373,
+      "rewards/margins": 0.23912616074085236,
+      "rewards/rejected": -1.3972079753875732,
+      "step": 4730
+    },
+    {
+      "epoch": 2.5342030439872887,
+      "grad_norm": 7.252170164669517,
+      "learning_rate": 7.149390750353779e-08,
+      "logits/chosen": -0.1420806497335434,
+      "logits/rejected": -0.2054506242275238,
+      "logps/chosen": -1.2775603532791138,
+      "logps/rejected": -1.4530556201934814,
+      "loss": 1.6186,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.2775603532791138,
+      "rewards/margins": 0.17549541592597961,
+      "rewards/rejected": -1.4530556201934814,
+      "step": 4735
+    },
+    {
+      "epoch": 2.5368790767686904,
+      "grad_norm": 9.076334442762303,
+      "learning_rate": 7.069346389974374e-08,
+      "logits/chosen": -0.24810704588890076,
+      "logits/rejected": -0.12928973138332367,
+      "logps/chosen": -1.2737659215927124,
+      "logps/rejected": -1.5296587944030762,
+      "loss": 1.6067,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.2737659215927124,
+      "rewards/margins": 0.2558927536010742,
+      "rewards/rejected": -1.5296587944030762,
+      "step": 4740
+    },
+    {
+      "epoch": 2.539555109550092,
+      "grad_norm": 7.678519243284076,
+      "learning_rate": 6.989718542984563e-08,
+      "logits/chosen": -0.19102391600608826,
+      "logits/rejected": -0.15975889563560486,
+      "logps/chosen": -1.2788324356079102,
+      "logps/rejected": -1.504441499710083,
+      "loss": 1.6004,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.2788324356079102,
+      "rewards/margins": 0.22560922801494598,
+      "rewards/rejected": -1.504441499710083,
+      "step": 4745
+    },
+    {
+      "epoch": 2.5422311423314934,
+      "grad_norm": 6.267154014115284,
+      "learning_rate": 6.9105079819341e-08,
+      "logits/chosen": -0.12780414521694183,
+      "logits/rejected": 0.04071787744760513,
+      "logps/chosen": -1.2187215089797974,
+      "logps/rejected": -1.6234019994735718,
+      "loss": 1.5236,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.2187215089797974,
+      "rewards/margins": 0.40468043088912964,
+      "rewards/rejected": -1.6234019994735718,
+      "step": 4750
+    },
+    {
+      "epoch": 2.544907175112895,
+      "grad_norm": 6.474135443347511,
+      "learning_rate": 6.831715475324163e-08,
+      "logits/chosen": -0.18726108968257904,
+      "logits/rejected": -0.04195536673069,
+      "logps/chosen": -1.1510288715362549,
+      "logps/rejected": -1.485931158065796,
+      "loss": 1.4969,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.1510288715362549,
+      "rewards/margins": 0.3349021375179291,
+      "rewards/rejected": -1.485931158065796,
+      "step": 4755
+    },
+    {
+      "epoch": 2.547583207894297,
+      "grad_norm": 7.059864002376292,
+      "learning_rate": 6.753341787600026e-08,
+      "logits/chosen": -0.250051885843277,
+      "logits/rejected": -0.12963862717151642,
+      "logps/chosen": -1.1672437191009521,
+      "logps/rejected": -1.3937098979949951,
+      "loss": 1.5341,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.1672437191009521,
+      "rewards/margins": 0.22646614909172058,
+      "rewards/rejected": -1.3937098979949951,
+      "step": 4760
+    },
+    {
+      "epoch": 2.5502592406756985,
+      "grad_norm": 8.424779242777895,
+      "learning_rate": 6.67538767914353e-08,
+      "logits/chosen": -0.272745817899704,
+      "logits/rejected": -0.11777077615261078,
+      "logps/chosen": -1.2558521032333374,
+      "logps/rejected": -1.4366075992584229,
+      "loss": 1.6214,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.2558521032333374,
+      "rewards/margins": 0.18075552582740784,
+      "rewards/rejected": -1.4366075992584229,
+      "step": 4765
+    },
+    {
+      "epoch": 2.5529352734571,
+      "grad_norm": 8.11145710217,
+      "learning_rate": 6.597853906265793e-08,
+      "logits/chosen": -0.22406180202960968,
+      "logits/rejected": -0.14268115162849426,
+      "logps/chosen": -1.2580957412719727,
+      "logps/rejected": -1.5128915309906006,
+      "loss": 1.5874,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.2580957412719727,
+      "rewards/margins": 0.25479573011398315,
+      "rewards/rejected": -1.5128915309906006,
+      "step": 4770
+    },
+    {
+      "epoch": 2.5556113062385015,
+      "grad_norm": 5.545354795700497,
+      "learning_rate": 6.5207412211998e-08,
+      "logits/chosen": -0.06541035324335098,
+      "logits/rejected": 0.02902628853917122,
+      "logps/chosen": -1.1863327026367188,
+      "logps/rejected": -1.4349019527435303,
+      "loss": 1.5328,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.1863327026367188,
+      "rewards/margins": 0.2485692948102951,
+      "rewards/rejected": -1.4349019527435303,
+      "step": 4775
+    },
+    {
+      "epoch": 2.558287339019903,
+      "grad_norm": 5.089746429603335,
+      "learning_rate": 6.444050372093186e-08,
+      "logits/chosen": -0.16423115134239197,
+      "logits/rejected": -0.09638829529285431,
+      "logps/chosen": -1.2353355884552002,
+      "logps/rejected": -1.4341075420379639,
+      "loss": 1.5856,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.2353355884552002,
+      "rewards/margins": 0.19877199828624725,
+      "rewards/rejected": -1.4341075420379639,
+      "step": 4780
+    },
+    {
+      "epoch": 2.5609633718013045,
+      "grad_norm": 10.491530025375786,
+      "learning_rate": 6.367782103000873e-08,
+      "logits/chosen": -0.15491080284118652,
+      "logits/rejected": -0.10681736469268799,
+      "logps/chosen": -1.2595205307006836,
+      "logps/rejected": -1.4247640371322632,
+      "loss": 1.6131,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.2595205307006836,
+      "rewards/margins": 0.1652435064315796,
+      "rewards/rejected": -1.4247640371322632,
+      "step": 4785
+    },
+    {
+      "epoch": 2.5636394045827062,
+      "grad_norm": 6.394223585792894,
+      "learning_rate": 6.29193715387798e-08,
+      "logits/chosen": -0.2331111878156662,
+      "logits/rejected": -0.10800018161535263,
+      "logps/chosen": -1.2580963373184204,
+      "logps/rejected": -1.5037890672683716,
+      "loss": 1.5824,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.2580963373184204,
+      "rewards/margins": 0.24569272994995117,
+      "rewards/rejected": -1.5037890672683716,
+      "step": 4790
+    },
+    {
+      "epoch": 2.566315437364108,
+      "grad_norm": 11.05058571995248,
+      "learning_rate": 6.216516260572502e-08,
+      "logits/chosen": -0.14371183514595032,
+      "logits/rejected": -0.09054752439260483,
+      "logps/chosen": -1.2728910446166992,
+      "logps/rejected": -1.4731087684631348,
+      "loss": 1.6238,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.2728910446166992,
+      "rewards/margins": 0.20021755993366241,
+      "rewards/rejected": -1.4731087684631348,
+      "step": 4795
+    },
+    {
+      "epoch": 2.568991470145509,
+      "grad_norm": 8.411984021616512,
+      "learning_rate": 6.141520154818297e-08,
+      "logits/chosen": -0.19774667918682098,
+      "logits/rejected": -0.10813900083303452,
+      "logps/chosen": -1.1911413669586182,
+      "logps/rejected": -1.4454268217086792,
+      "loss": 1.5444,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.1911413669586182,
+      "rewards/margins": 0.25428539514541626,
+      "rewards/rejected": -1.4454268217086792,
+      "step": 4800
+    },
+    {
+      "epoch": 2.568991470145509,
+      "eval_logits/chosen": 0.069010429084301,
+      "eval_logits/rejected": 0.1403309553861618,
+      "eval_logps/chosen": -1.303430199623108,
+      "eval_logps/rejected": -1.5013798475265503,
+      "eval_loss": 1.6473349332809448,
+      "eval_rewards/accuracies": 0.5719584822654724,
+      "eval_rewards/chosen": -1.303430199623108,
+      "eval_rewards/margins": 0.19794969260692596,
+      "eval_rewards/rejected": -1.5013798475265503,
+      "eval_runtime": 40.9803,
+      "eval_samples_per_second": 32.821,
+      "eval_steps_per_second": 8.223,
+      "step": 4800
+    },
+    {
+      "epoch": 2.571667502926911,
+      "grad_norm": 9.907133386402505,
+      "learning_rate": 6.066949564227897e-08,
+      "logits/chosen": -0.2680151164531708,
+      "logits/rejected": -0.1689799576997757,
+      "logps/chosen": -1.2977831363677979,
+      "logps/rejected": -1.565671682357788,
+      "loss": 1.6163,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.2977831363677979,
+      "rewards/margins": 0.2678885757923126,
+      "rewards/rejected": -1.565671682357788,
+      "step": 4805
+    },
+    {
+      "epoch": 2.574343535708312,
+      "grad_norm": 6.806730281339653,
+      "learning_rate": 5.992805212285523e-08,
+      "logits/chosen": -0.14087334275245667,
+      "logits/rejected": -0.02274361439049244,
+      "logps/chosen": -1.3015844821929932,
+      "logps/rejected": -1.5247794389724731,
+      "loss": 1.6321,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.3015844821929932,
+      "rewards/margins": 0.223194882273674,
+      "rewards/rejected": -1.5247794389724731,
+      "step": 4810
+    },
+    {
+      "epoch": 2.577019568489714,
+      "grad_norm": 8.51556092180203,
+      "learning_rate": 5.9190878183399684e-08,
+      "logits/chosen": -0.1476089209318161,
+      "logits/rejected": -0.04703466221690178,
+      "logps/chosen": -1.146867036819458,
+      "logps/rejected": -1.5179466009140015,
+      "loss": 1.4784,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.146867036819458,
+      "rewards/margins": 0.37107959389686584,
+      "rewards/rejected": -1.5179466009140015,
+      "step": 4815
+    },
+    {
+      "epoch": 2.5796956012711156,
+      "grad_norm": 8.72271523604443,
+      "learning_rate": 5.845798097597748e-08,
+      "logits/chosen": -0.14900721609592438,
+      "logits/rejected": -0.0796498954296112,
+      "logps/chosen": -1.3060250282287598,
+      "logps/rejected": -1.4967185258865356,
+      "loss": 1.6296,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.3060250282287598,
+      "rewards/margins": 0.1906934529542923,
+      "rewards/rejected": -1.4967185258865356,
+      "step": 4820
+    },
+    {
+      "epoch": 2.5823716340525174,
+      "grad_norm": 10.002189649313841,
+      "learning_rate": 5.772936761116026e-08,
+      "logits/chosen": -0.14325377345085144,
+      "logits/rejected": -0.05125095695257187,
+      "logps/chosen": -1.20247220993042,
+      "logps/rejected": -1.3964070081710815,
+      "loss": 1.5706,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.20247220993042,
+      "rewards/margins": 0.19393488764762878,
+      "rewards/rejected": -1.3964070081710815,
+      "step": 4825
+    },
+    {
+      "epoch": 2.5850476668339186,
+      "grad_norm": 5.577297169590059,
+      "learning_rate": 5.700504515795829e-08,
+      "logits/chosen": -0.18866679072380066,
+      "logits/rejected": -0.06108071282505989,
+      "logps/chosen": -1.287221908569336,
+      "logps/rejected": -1.4399975538253784,
+      "loss": 1.6316,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.287221908569336,
+      "rewards/margins": 0.1527756154537201,
+      "rewards/rejected": -1.4399975538253784,
+      "step": 4830
+    },
+    {
+      "epoch": 2.5877236996153203,
+      "grad_norm": 10.808848370754,
+      "learning_rate": 5.628502064375101e-08,
+      "logits/chosen": -0.3119337558746338,
+      "logits/rejected": -0.15793856978416443,
+      "logps/chosen": -1.2294279336929321,
+      "logps/rejected": -1.4302443265914917,
+      "loss": 1.5851,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.2294279336929321,
+      "rewards/margins": 0.2008165419101715,
+      "rewards/rejected": -1.4302443265914917,
+      "step": 4835
+    },
+    {
+      "epoch": 2.5903997323967216,
+      "grad_norm": 17.377615026752256,
+      "learning_rate": 5.55693010542197e-08,
+      "logits/chosen": -0.26505112648010254,
+      "logits/rejected": -0.11424796283245087,
+      "logps/chosen": -1.2466034889221191,
+      "logps/rejected": -1.443650484085083,
+      "loss": 1.5895,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.2466034889221191,
+      "rewards/margins": 0.19704698026180267,
+      "rewards/rejected": -1.443650484085083,
+      "step": 4840
+    },
+    {
+      "epoch": 2.5930757651781233,
+      "grad_norm": 6.862180399559727,
+      "learning_rate": 5.485789333327856e-08,
+      "logits/chosen": -0.23199248313903809,
+      "logits/rejected": -0.1249762773513794,
+      "logps/chosen": -1.20241379737854,
+      "logps/rejected": -1.4607778787612915,
+      "loss": 1.531,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.20241379737854,
+      "rewards/margins": 0.25836381316185,
+      "rewards/rejected": -1.4607778787612915,
+      "step": 4845
+    },
+    {
+      "epoch": 2.595751797959525,
+      "grad_norm": 11.417551410862856,
+      "learning_rate": 5.4150804383008675e-08,
+      "logits/chosen": -0.3245030343532562,
+      "logits/rejected": -0.18769951164722443,
+      "logps/chosen": -1.1993231773376465,
+      "logps/rejected": -1.4456372261047363,
+      "loss": 1.5505,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.1993231773376465,
+      "rewards/margins": 0.24631407856941223,
+      "rewards/rejected": -1.4456372261047363,
+      "step": 4850
+    },
+    {
+      "epoch": 2.5984278307409268,
+      "grad_norm": 10.991552356367377,
+      "learning_rate": 5.344804106359002e-08,
+      "logits/chosen": -0.1707092821598053,
+      "logits/rejected": -0.040122199803590775,
+      "logps/chosen": -1.1905627250671387,
+      "logps/rejected": -1.3950374126434326,
+      "loss": 1.5777,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.1905627250671387,
+      "rewards/margins": 0.204474538564682,
+      "rewards/rejected": -1.3950374126434326,
+      "step": 4855
+    },
+    {
+      "epoch": 2.601103863522328,
+      "grad_norm": 10.67200349867524,
+      "learning_rate": 5.274961019323559e-08,
+      "logits/chosen": -0.20811323821544647,
+      "logits/rejected": -0.15802529454231262,
+      "logps/chosen": -1.1508222818374634,
+      "logps/rejected": -1.5101139545440674,
+      "loss": 1.4898,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.1508222818374634,
+      "rewards/margins": 0.3592917025089264,
+      "rewards/rejected": -1.5101139545440674,
+      "step": 4860
+    },
+    {
+      "epoch": 2.6037798963037297,
+      "grad_norm": 7.816821358452244,
+      "learning_rate": 5.205551854812451e-08,
+      "logits/chosen": -0.2871929407119751,
+      "logits/rejected": -0.21601462364196777,
+      "logps/chosen": -1.2162559032440186,
+      "logps/rejected": -1.519219994544983,
+      "loss": 1.5526,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.2162559032440186,
+      "rewards/margins": 0.3029640316963196,
+      "rewards/rejected": -1.519219994544983,
+      "step": 4865
+    },
+    {
+      "epoch": 2.606455929085131,
+      "grad_norm": 10.124198148417449,
+      "learning_rate": 5.1365772862337177e-08,
+      "logits/chosen": -0.16407471895217896,
+      "logits/rejected": -0.0544242337346077,
+      "logps/chosen": -1.2421973943710327,
+      "logps/rejected": -1.4441970586776733,
+      "loss": 1.5994,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.2421973943710327,
+      "rewards/margins": 0.20199963450431824,
+      "rewards/rejected": -1.4441970586776733,
+      "step": 4870
+    },
+    {
+      "epoch": 2.6091319618665327,
+      "grad_norm": 8.284000148873433,
+      "learning_rate": 5.068037982778905e-08,
+      "logits/chosen": -0.03732685372233391,
+      "logits/rejected": 0.022808223962783813,
+      "logps/chosen": -1.2028716802597046,
+      "logps/rejected": -1.5486289262771606,
+      "loss": 1.5438,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.2028716802597046,
+      "rewards/margins": 0.34575721621513367,
+      "rewards/rejected": -1.5486289262771606,
+      "step": 4875
+    },
+    {
+      "epoch": 2.6118079946479344,
+      "grad_norm": 9.236354556919212,
+      "learning_rate": 4.999934609416656e-08,
+      "logits/chosen": -0.11083243042230606,
+      "logits/rejected": -0.01954919472336769,
+      "logps/chosen": -1.1937742233276367,
+      "logps/rejected": -1.4890735149383545,
+      "loss": 1.5265,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.1937742233276367,
+      "rewards/margins": 0.29529935121536255,
+      "rewards/rejected": -1.4890735149383545,
+      "step": 4880
+    },
+    {
+      "epoch": 2.614484027429336,
+      "grad_norm": 8.007453309916604,
+      "learning_rate": 4.932267826886183e-08,
+      "logits/chosen": -0.12022782862186432,
+      "logits/rejected": -0.05459488555788994,
+      "logps/chosen": -1.3056491613388062,
+      "logps/rejected": -1.5686249732971191,
+      "loss": 1.6349,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.3056491613388062,
+      "rewards/margins": 0.26297587156295776,
+      "rewards/rejected": -1.5686249732971191,
+      "step": 4885
+    },
+    {
+      "epoch": 2.6171600602107374,
+      "grad_norm": 9.567757022684548,
+      "learning_rate": 4.8650382916909206e-08,
+      "logits/chosen": -0.2669495940208435,
+      "logits/rejected": -0.12905225157737732,
+      "logps/chosen": -1.235668420791626,
+      "logps/rejected": -1.5042340755462646,
+      "loss": 1.5718,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.235668420791626,
+      "rewards/margins": 0.268565833568573,
+      "rewards/rejected": -1.5042340755462646,
+      "step": 4890
+    },
+    {
+      "epoch": 2.619836092992139,
+      "grad_norm": 6.9529581708975625,
+      "learning_rate": 4.7982466560920976e-08,
+      "logits/chosen": -0.19365879893302917,
+      "logits/rejected": -0.11342108249664307,
+      "logps/chosen": -1.2696317434310913,
+      "logps/rejected": -1.4244247674942017,
+      "loss": 1.6236,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.2696317434310913,
+      "rewards/margins": 0.1547931283712387,
+      "rewards/rejected": -1.4244247674942017,
+      "step": 4895
+    },
+    {
+      "epoch": 2.622512125773541,
+      "grad_norm": 6.010126890420108,
+      "learning_rate": 4.7318935681024685e-08,
+      "logits/chosen": -0.14299534261226654,
+      "logits/rejected": -0.024873051792383194,
+      "logps/chosen": -1.2177270650863647,
+      "logps/rejected": -1.4972065687179565,
+      "loss": 1.5518,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.2177270650863647,
+      "rewards/margins": 0.2794795632362366,
+      "rewards/rejected": -1.4972065687179565,
+      "step": 4900
+    },
+    {
+      "epoch": 2.625188158554942,
+      "grad_norm": 6.432348217455601,
+      "learning_rate": 4.6659796714799745e-08,
+      "logits/chosen": -0.19225557148456573,
+      "logits/rejected": -0.06428401172161102,
+      "logps/chosen": -1.2267589569091797,
+      "logps/rejected": -1.492279291152954,
+      "loss": 1.5688,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.2267589569091797,
+      "rewards/margins": 0.2655201852321625,
+      "rewards/rejected": -1.492279291152954,
+      "step": 4905
+    },
+    {
+      "epoch": 2.627864191336344,
+      "grad_norm": 7.250219709550545,
+      "learning_rate": 4.60050560572155e-08,
+      "logits/chosen": -0.2531941831111908,
+      "logits/rejected": -0.2812545895576477,
+      "logps/chosen": -1.2359212636947632,
+      "logps/rejected": -1.5699865818023682,
+      "loss": 1.5642,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.2359212636947632,
+      "rewards/margins": 0.33406534790992737,
+      "rewards/rejected": -1.5699865818023682,
+      "step": 4910
+    },
+    {
+      "epoch": 2.6305402241177456,
+      "grad_norm": 8.31195879409968,
+      "learning_rate": 4.535472006056834e-08,
+      "logits/chosen": -0.16209043562412262,
+      "logits/rejected": -0.08424313366413116,
+      "logps/chosen": -1.151822566986084,
+      "logps/rejected": -1.447588324546814,
+      "loss": 1.4963,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.151822566986084,
+      "rewards/margins": 0.2957659363746643,
+      "rewards/rejected": -1.447588324546814,
+      "step": 4915
+    },
+    {
+      "epoch": 2.6332162568991473,
+      "grad_norm": 9.425398992684931,
+      "learning_rate": 4.470879503442132e-08,
+      "logits/chosen": -0.1478452831506729,
+      "logits/rejected": -0.09613370895385742,
+      "logps/chosen": -1.2299785614013672,
+      "logps/rejected": -1.4136711359024048,
+      "loss": 1.5997,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.2299785614013672,
+      "rewards/margins": 0.18369272351264954,
+      "rewards/rejected": -1.4136711359024048,
+      "step": 4920
+    },
+    {
+      "epoch": 2.6358922896805486,
+      "grad_norm": 8.625771505341113,
+      "learning_rate": 4.406728724554154e-08,
+      "logits/chosen": -0.3559058904647827,
+      "logits/rejected": -0.14755351841449738,
+      "logps/chosen": -1.1982418298721313,
+      "logps/rejected": -1.4626787900924683,
+      "loss": 1.5445,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.1982418298721313,
+      "rewards/margins": 0.26443710923194885,
+      "rewards/rejected": -1.4626787900924683,
+      "step": 4925
+    },
+    {
+      "epoch": 2.6385683224619503,
+      "grad_norm": 6.369269401234845,
+      "learning_rate": 4.3430202917840664e-08,
+      "logits/chosen": -0.12286825478076935,
+      "logits/rejected": -0.0016535192262381315,
+      "logps/chosen": -1.274578332901001,
+      "logps/rejected": -1.5974867343902588,
+      "loss": 1.5924,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.274578332901001,
+      "rewards/margins": 0.32290831208229065,
+      "rewards/rejected": -1.5974867343902588,
+      "step": 4930
+    },
+    {
+      "epoch": 2.6412443552433515,
+      "grad_norm": 10.84824946742521,
+      "learning_rate": 4.279754823231346e-08,
+      "logits/chosen": -0.25615420937538147,
+      "logits/rejected": -0.1267656832933426,
+      "logps/chosen": -1.2327163219451904,
+      "logps/rejected": -1.4325826168060303,
+      "loss": 1.5876,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.2327163219451904,
+      "rewards/margins": 0.19986644387245178,
+      "rewards/rejected": -1.4325826168060303,
+      "step": 4935
+    },
+    {
+      "epoch": 2.6439203880247533,
+      "grad_norm": 6.695587205505035,
+      "learning_rate": 4.216932932697859e-08,
+      "logits/chosen": -0.19970975816249847,
+      "logits/rejected": -0.14457407593727112,
+      "logps/chosen": -1.2329965829849243,
+      "logps/rejected": -1.37167489528656,
+      "loss": 1.5926,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.2329965829849243,
+      "rewards/margins": 0.13867846131324768,
+      "rewards/rejected": -1.37167489528656,
+      "step": 4940
+    },
+    {
+      "epoch": 2.646596420806155,
+      "grad_norm": 7.353167999405972,
+      "learning_rate": 4.154555229681844e-08,
+      "logits/chosen": -0.18768291175365448,
+      "logits/rejected": -0.010452374815940857,
+      "logps/chosen": -1.2278202772140503,
+      "logps/rejected": -1.536572813987732,
+      "loss": 1.5483,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.2278202772140503,
+      "rewards/margins": 0.3087523877620697,
+      "rewards/rejected": -1.536572813987732,
+      "step": 4945
+    },
+    {
+      "epoch": 2.6492724535875567,
+      "grad_norm": 12.227792809999936,
+      "learning_rate": 4.092622319372069e-08,
+      "logits/chosen": -0.22052931785583496,
+      "logits/rejected": -0.12232838571071625,
+      "logps/chosen": -1.1905124187469482,
+      "logps/rejected": -1.4453401565551758,
+      "loss": 1.5503,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.1905124187469482,
+      "rewards/margins": 0.2548278272151947,
+      "rewards/rejected": -1.4453401565551758,
+      "step": 4950
+    },
+    {
+      "epoch": 2.651948486368958,
+      "grad_norm": 8.578760957784235,
+      "learning_rate": 4.031134802641889e-08,
+      "logits/chosen": -0.21375855803489685,
+      "logits/rejected": -0.22308556735515594,
+      "logps/chosen": -1.2271889448165894,
+      "logps/rejected": -1.5128446817398071,
+      "loss": 1.5502,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.2271889448165894,
+      "rewards/margins": 0.2856557071208954,
+      "rewards/rejected": -1.5128446817398071,
+      "step": 4955
+    },
+    {
+      "epoch": 2.6546245191503597,
+      "grad_norm": 7.265117101221275,
+      "learning_rate": 3.970093276043468e-08,
+      "logits/chosen": -0.11913114786148071,
+      "logits/rejected": -0.041926462203264236,
+      "logps/chosen": -1.2388899326324463,
+      "logps/rejected": -1.4442875385284424,
+      "loss": 1.5851,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.2388899326324463,
+      "rewards/margins": 0.20539765059947968,
+      "rewards/rejected": -1.4442875385284424,
+      "step": 4960
+    },
+    {
+      "epoch": 2.657300551931761,
+      "grad_norm": 8.696253922209147,
+      "learning_rate": 3.9094983318019584e-08,
+      "logits/chosen": -0.23500557243824005,
+      "logits/rejected": -0.12808677554130554,
+      "logps/chosen": -1.140141487121582,
+      "logps/rejected": -1.3926841020584106,
+      "loss": 1.4881,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.140141487121582,
+      "rewards/margins": 0.2525426745414734,
+      "rewards/rejected": -1.3926841020584106,
+      "step": 4965
+    },
+    {
+      "epoch": 2.6599765847131627,
+      "grad_norm": 11.446705748719662,
+      "learning_rate": 3.849350557809789e-08,
+      "logits/chosen": -0.13588054478168488,
+      "logits/rejected": -0.08402448147535324,
+      "logps/chosen": -1.1959439516067505,
+      "logps/rejected": -1.4947633743286133,
+      "loss": 1.5293,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.1959439516067505,
+      "rewards/margins": 0.29881924390792847,
+      "rewards/rejected": -1.4947633743286133,
+      "step": 4970
+    },
+    {
+      "epoch": 2.6626526174945644,
+      "grad_norm": 8.997039686082442,
+      "learning_rate": 3.789650537620903e-08,
+      "logits/chosen": -0.21160972118377686,
+      "logits/rejected": -0.17673712968826294,
+      "logps/chosen": -1.2614823579788208,
+      "logps/rejected": -1.4655091762542725,
+      "loss": 1.6031,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.2614823579788208,
+      "rewards/margins": 0.2040266990661621,
+      "rewards/rejected": -1.4655091762542725,
+      "step": 4975
+    },
+    {
+      "epoch": 2.665328650275966,
+      "grad_norm": 10.466268389357104,
+      "learning_rate": 3.730398850445182e-08,
+      "logits/chosen": -0.09437865763902664,
+      "logits/rejected": -0.035064686089754105,
+      "logps/chosen": -1.3431638479232788,
+      "logps/rejected": -1.5557583570480347,
+      "loss": 1.6559,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.3431638479232788,
+      "rewards/margins": 0.21259450912475586,
+      "rewards/rejected": -1.5557583570480347,
+      "step": 4980
+    },
+    {
+      "epoch": 2.6680046830573674,
+      "grad_norm": 11.519904973680102,
+      "learning_rate": 3.671596071142735e-08,
+      "logits/chosen": -0.14721126854419708,
+      "logits/rejected": -0.006001046393066645,
+      "logps/chosen": -1.1870566606521606,
+      "logps/rejected": -1.4658606052398682,
+      "loss": 1.5466,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.1870566606521606,
+      "rewards/margins": 0.2788039743900299,
+      "rewards/rejected": -1.4658606052398682,
+      "step": 4985
+    },
+    {
+      "epoch": 2.670680715838769,
+      "grad_norm": 10.178088788076465,
+      "learning_rate": 3.6132427702183996e-08,
+      "logits/chosen": -0.26032769680023193,
+      "logits/rejected": -0.0708533450961113,
+      "logps/chosen": -1.1861783266067505,
+      "logps/rejected": -1.4879579544067383,
+      "loss": 1.5326,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.1861783266067505,
+      "rewards/margins": 0.301779568195343,
+      "rewards/rejected": -1.4879579544067383,
+      "step": 4990
+    },
+    {
+      "epoch": 2.6733567486201704,
+      "grad_norm": 9.76883783497797,
+      "learning_rate": 3.555339513816147e-08,
+      "logits/chosen": -0.23129093647003174,
+      "logits/rejected": -0.223618745803833,
+      "logps/chosen": -1.2377936840057373,
+      "logps/rejected": -1.4420830011367798,
+      "loss": 1.5989,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.2377936840057373,
+      "rewards/margins": 0.20428943634033203,
+      "rewards/rejected": -1.4420830011367798,
+      "step": 4995
+    },
+    {
+      "epoch": 2.676032781401572,
+      "grad_norm": 8.338276783853692,
+      "learning_rate": 3.497886863713639e-08,
+      "logits/chosen": -0.24142029881477356,
+      "logits/rejected": -0.23088951408863068,
+      "logps/chosen": -1.2321035861968994,
+      "logps/rejected": -1.5279680490493774,
+      "loss": 1.5747,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.2321035861968994,
+      "rewards/margins": 0.295864462852478,
+      "rewards/rejected": -1.5279680490493774,
+      "step": 5000
+    },
+    {
+      "epoch": 2.678708814182974,
+      "grad_norm": 9.870028644774022,
+      "learning_rate": 3.440885377316721e-08,
+      "logits/chosen": -0.1602504551410675,
+      "logits/rejected": -0.12937873601913452,
+      "logps/chosen": -1.220651388168335,
+      "logps/rejected": -1.4318891763687134,
+      "loss": 1.562,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.220651388168335,
+      "rewards/margins": 0.21123778820037842,
+      "rewards/rejected": -1.4318891763687134,
+      "step": 5005
+    },
+    {
+      "epoch": 2.6813848469643755,
+      "grad_norm": 9.349431556525046,
+      "learning_rate": 3.384335607654082e-08,
+      "logits/chosen": -0.13725784420967102,
+      "logits/rejected": -0.06370799988508224,
+      "logps/chosen": -1.3512427806854248,
+      "logps/rejected": -1.5464354753494263,
+      "loss": 1.6933,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.3512427806854248,
+      "rewards/margins": 0.1951926350593567,
+      "rewards/rejected": -1.5464354753494263,
+      "step": 5010
+    },
+    {
+      "epoch": 2.684060879745777,
+      "grad_norm": 10.099696408988086,
+      "learning_rate": 3.328238103371811e-08,
+      "logits/chosen": -0.25667843222618103,
+      "logits/rejected": -0.20781448483467102,
+      "logps/chosen": -1.2320544719696045,
+      "logps/rejected": -1.5089162588119507,
+      "loss": 1.573,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.2320544719696045,
+      "rewards/margins": 0.2768619656562805,
+      "rewards/rejected": -1.5089162588119507,
+      "step": 5015
+    },
+    {
+      "epoch": 2.6867369125271785,
+      "grad_norm": 8.636269338621892,
+      "learning_rate": 3.272593408728169e-08,
+      "logits/chosen": -0.25457826256752014,
+      "logits/rejected": -0.09930596500635147,
+      "logps/chosen": -1.2071006298065186,
+      "logps/rejected": -1.385921835899353,
+      "loss": 1.5817,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.2071006298065186,
+      "rewards/margins": 0.1788211315870285,
+      "rewards/rejected": -1.385921835899353,
+      "step": 5020
+    },
+    {
+      "epoch": 2.6894129453085798,
+      "grad_norm": 7.812458444564327,
+      "learning_rate": 3.217402063588204e-08,
+      "logits/chosen": -0.25639787316322327,
+      "logits/rejected": -0.14365923404693604,
+      "logps/chosen": -1.263940691947937,
+      "logps/rejected": -1.4570047855377197,
+      "loss": 1.6064,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.263940691947937,
+      "rewards/margins": 0.19306401908397675,
+      "rewards/rejected": -1.4570047855377197,
+      "step": 5025
+    },
+    {
+      "epoch": 2.6920889780899815,
+      "grad_norm": 8.15837912990784,
+      "learning_rate": 3.162664603418608e-08,
+      "logits/chosen": -0.18479934334754944,
+      "logits/rejected": -0.13748101890087128,
+      "logps/chosen": -1.2296315431594849,
+      "logps/rejected": -1.449176549911499,
+      "loss": 1.5842,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.2296315431594849,
+      "rewards/margins": 0.21954509615898132,
+      "rewards/rejected": -1.449176549911499,
+      "step": 5030
+    },
+    {
+      "epoch": 2.694765010871383,
+      "grad_norm": 14.596781404528606,
+      "learning_rate": 3.1083815592824416e-08,
+      "logits/chosen": -0.23647980391979218,
+      "logits/rejected": -0.12656892836093903,
+      "logps/chosen": -1.2724589109420776,
+      "logps/rejected": -1.484289526939392,
+      "loss": 1.6234,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.2724589109420776,
+      "rewards/margins": 0.2118304967880249,
+      "rewards/rejected": -1.484289526939392,
+      "step": 5035
+    },
+    {
+      "epoch": 2.697441043652785,
+      "grad_norm": 11.529255673618911,
+      "learning_rate": 3.054553457834053e-08,
+      "logits/chosen": -0.01275326032191515,
+      "logits/rejected": -0.06487753987312317,
+      "logps/chosen": -1.1942696571350098,
+      "logps/rejected": -1.5052051544189453,
+      "loss": 1.5246,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.1942696571350098,
+      "rewards/margins": 0.31093549728393555,
+      "rewards/rejected": -1.5052051544189453,
+      "step": 5040
+    },
+    {
+      "epoch": 2.700117076434186,
+      "grad_norm": 8.754850568076293,
+      "learning_rate": 3.0011808213139036e-08,
+      "logits/chosen": -0.1731742024421692,
+      "logits/rejected": -0.14932702481746674,
+      "logps/chosen": -1.1981568336486816,
+      "logps/rejected": -1.4055202007293701,
+      "loss": 1.5489,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.1981568336486816,
+      "rewards/margins": 0.20736345648765564,
+      "rewards/rejected": -1.4055202007293701,
+      "step": 5045
+    },
+    {
+      "epoch": 2.702793109215588,
+      "grad_norm": 8.859370600902391,
+      "learning_rate": 2.948264167543568e-08,
+      "logits/chosen": -0.20417948067188263,
+      "logits/rejected": -0.16572143137454987,
+      "logps/chosen": -1.1094834804534912,
+      "logps/rejected": -1.4431778192520142,
+      "loss": 1.4449,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.1094834804534912,
+      "rewards/margins": 0.33369430899620056,
+      "rewards/rejected": -1.4431778192520142,
+      "step": 5050
+    },
+    {
+      "epoch": 2.7054691419969896,
+      "grad_norm": 8.30519925014008,
+      "learning_rate": 2.8958040099206216e-08,
+      "logits/chosen": -0.30399882793426514,
+      "logits/rejected": -0.23754310607910156,
+      "logps/chosen": -1.1653364896774292,
+      "logps/rejected": -1.4400817155838013,
+      "loss": 1.5383,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.1653364896774292,
+      "rewards/margins": 0.2747451663017273,
+      "rewards/rejected": -1.4400817155838013,
+      "step": 5055
+    },
+    {
+      "epoch": 2.708145174778391,
+      "grad_norm": 11.704168087582307,
+      "learning_rate": 2.843800857413775e-08,
+      "logits/chosen": -0.18940813839435577,
+      "logits/rejected": -0.14188504219055176,
+      "logps/chosen": -1.2219481468200684,
+      "logps/rejected": -1.4544917345046997,
+      "loss": 1.5719,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.2219481468200684,
+      "rewards/margins": 0.23254361748695374,
+      "rewards/rejected": -1.4544917345046997,
+      "step": 5060
+    },
+    {
+      "epoch": 2.7108212075597926,
+      "grad_norm": 9.32374694815667,
+      "learning_rate": 2.7922552145578203e-08,
+      "logits/chosen": -0.2245454341173172,
+      "logits/rejected": -0.03208258002996445,
+      "logps/chosen": -1.2869073152542114,
+      "logps/rejected": -1.5468248128890991,
+      "loss": 1.6126,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.2869073152542114,
+      "rewards/margins": 0.2599175274372101,
+      "rewards/rejected": -1.5468248128890991,
+      "step": 5065
+    },
+    {
+      "epoch": 2.7134972403411943,
+      "grad_norm": 7.559388414982109,
+      "learning_rate": 2.7411675814488277e-08,
+      "logits/chosen": -0.10575256496667862,
+      "logits/rejected": 0.030589479953050613,
+      "logps/chosen": -1.1588886976242065,
+      "logps/rejected": -1.376731514930725,
+      "loss": 1.5377,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.1588886976242065,
+      "rewards/margins": 0.21784265339374542,
+      "rewards/rejected": -1.376731514930725,
+      "step": 5070
+    },
+    {
+      "epoch": 2.7161732731225956,
+      "grad_norm": 10.956395790032781,
+      "learning_rate": 2.690538453739216e-08,
+      "logits/chosen": -0.18339474499225616,
+      "logits/rejected": -0.12223565578460693,
+      "logps/chosen": -1.2097294330596924,
+      "logps/rejected": -1.341022253036499,
+      "loss": 1.6031,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.2097294330596924,
+      "rewards/margins": 0.13129296898841858,
+      "rewards/rejected": -1.341022253036499,
+      "step": 5075
+    },
+    {
+      "epoch": 2.7188493059039973,
+      "grad_norm": 7.363109836236882,
+      "learning_rate": 2.6403683226330298e-08,
+      "logits/chosen": -0.2401486337184906,
+      "logits/rejected": -0.13127775490283966,
+      "logps/chosen": -1.2772992849349976,
+      "logps/rejected": -1.4677261114120483,
+      "loss": 1.6174,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.2772992849349976,
+      "rewards/margins": 0.1904267817735672,
+      "rewards/rejected": -1.4677261114120483,
+      "step": 5080
+    },
+    {
+      "epoch": 2.721525338685399,
+      "grad_norm": 10.174103894659988,
+      "learning_rate": 2.5906576748810804e-08,
+      "logits/chosen": -0.3042605221271515,
+      "logits/rejected": -0.20611527562141418,
+      "logps/chosen": -1.1317962408065796,
+      "logps/rejected": -1.5567015409469604,
+      "loss": 1.4574,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.1317962408065796,
+      "rewards/margins": 0.42490530014038086,
+      "rewards/rejected": -1.5567015409469604,
+      "step": 5085
+    },
+    {
+      "epoch": 2.7242013714668003,
+      "grad_norm": 11.486051703605545,
+      "learning_rate": 2.5414069927763016e-08,
+      "logits/chosen": -0.28841036558151245,
+      "logits/rejected": -0.16781942546367645,
+      "logps/chosen": -1.2877984046936035,
+      "logps/rejected": -1.5316404104232788,
+      "loss": 1.6117,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.2877984046936035,
+      "rewards/margins": 0.24384205043315887,
+      "rewards/rejected": -1.5316404104232788,
+      "step": 5090
+    },
+    {
+      "epoch": 2.726877404248202,
+      "grad_norm": 6.742605643992969,
+      "learning_rate": 2.4926167541490185e-08,
+      "logits/chosen": -0.31649142503738403,
+      "logits/rejected": -0.1508931666612625,
+      "logps/chosen": -1.2179322242736816,
+      "logps/rejected": -1.5233149528503418,
+      "loss": 1.5515,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.2179322242736816,
+      "rewards/margins": 0.30538275837898254,
+      "rewards/rejected": -1.5233149528503418,
+      "step": 5095
+    },
+    {
+      "epoch": 2.7295534370296037,
+      "grad_norm": 9.89213418989225,
+      "learning_rate": 2.4442874323623574e-08,
+      "logits/chosen": -0.11926839500665665,
+      "logits/rejected": -0.006692640483379364,
+      "logps/chosen": -1.1790974140167236,
+      "logps/rejected": -1.5305302143096924,
+      "loss": 1.5145,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -1.1790974140167236,
+      "rewards/margins": 0.35143283009529114,
+      "rewards/rejected": -1.5305302143096924,
+      "step": 5100
+    },
+    {
+      "epoch": 2.7322294698110055,
+      "grad_norm": 10.34193039811828,
+      "learning_rate": 2.396419496307589e-08,
+      "logits/chosen": -0.18230049312114716,
+      "logits/rejected": -0.044870562851428986,
+      "logps/chosen": -1.2445478439331055,
+      "logps/rejected": -1.4135087728500366,
+      "loss": 1.5967,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.2445478439331055,
+      "rewards/margins": 0.16896098852157593,
+      "rewards/rejected": -1.4135087728500366,
+      "step": 5105
+    },
+    {
+      "epoch": 2.7349055025924067,
+      "grad_norm": 8.068889265098178,
+      "learning_rate": 2.349013410399653e-08,
+      "logits/chosen": -0.2620542645454407,
+      "logits/rejected": -0.1357966959476471,
+      "logps/chosen": -1.204101324081421,
+      "logps/rejected": -1.4400665760040283,
+      "loss": 1.5632,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.204101324081421,
+      "rewards/margins": 0.23596513271331787,
+      "rewards/rejected": -1.4400665760040283,
+      "step": 5110
+    },
+    {
+      "epoch": 2.7375815353738084,
+      "grad_norm": 8.335383192965962,
+      "learning_rate": 2.3020696345725954e-08,
+      "logits/chosen": -0.2606256902217865,
+      "logits/rejected": -0.08713199198246002,
+      "logps/chosen": -1.2145345211029053,
+      "logps/rejected": -1.5258738994598389,
+      "loss": 1.539,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.2145345211029053,
+      "rewards/margins": 0.31133946776390076,
+      "rewards/rejected": -1.5258738994598389,
+      "step": 5115
+    },
+    {
+      "epoch": 2.7402575681552097,
+      "grad_norm": 10.889216618276286,
+      "learning_rate": 2.2555886242751398e-08,
+      "logits/chosen": -0.20491114258766174,
+      "logits/rejected": -0.16528639197349548,
+      "logps/chosen": -1.3192756175994873,
+      "logps/rejected": -1.5630297660827637,
+      "loss": 1.6264,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.3192756175994873,
+      "rewards/margins": 0.24375399947166443,
+      "rewards/rejected": -1.5630297660827637,
+      "step": 5120
+    },
+    {
+      "epoch": 2.7429336009366114,
+      "grad_norm": 15.88467586253665,
+      "learning_rate": 2.2095708304662453e-08,
+      "logits/chosen": -0.29381468892097473,
+      "logits/rejected": -0.09942570328712463,
+      "logps/chosen": -1.1907604932785034,
+      "logps/rejected": -1.4625751972198486,
+      "loss": 1.5411,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.1907604932785034,
+      "rewards/margins": 0.2718146741390228,
+      "rewards/rejected": -1.4625751972198486,
+      "step": 5125
+    },
+    {
+      "epoch": 2.745609633718013,
+      "grad_norm": 6.842095804156989,
+      "learning_rate": 2.16401669961076e-08,
+      "logits/chosen": -0.3690885305404663,
+      "logits/rejected": -0.2036101520061493,
+      "logps/chosen": -1.2096027135849,
+      "logps/rejected": -1.5022227764129639,
+      "loss": 1.5295,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.2096027135849,
+      "rewards/margins": 0.2926200032234192,
+      "rewards/rejected": -1.5022227764129639,
+      "step": 5130
+    },
+    {
+      "epoch": 2.748285666499415,
+      "grad_norm": 8.580770840370676,
+      "learning_rate": 2.1189266736750532e-08,
+      "logits/chosen": -0.130879208445549,
+      "logits/rejected": -0.0750630795955658,
+      "logps/chosen": -1.209316372871399,
+      "logps/rejected": -1.454590916633606,
+      "loss": 1.5721,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.209316372871399,
+      "rewards/margins": 0.24527454376220703,
+      "rewards/rejected": -1.454590916633606,
+      "step": 5135
+    },
+    {
+      "epoch": 2.750961699280816,
+      "grad_norm": 6.706691055358692,
+      "learning_rate": 2.0743011901227623e-08,
+      "logits/chosen": -0.1535293161869049,
+      "logits/rejected": -0.04885602742433548,
+      "logps/chosen": -1.2943406105041504,
+      "logps/rejected": -1.476296305656433,
+      "loss": 1.6434,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.2943406105041504,
+      "rewards/margins": 0.1819557249546051,
+      "rewards/rejected": -1.476296305656433,
+      "step": 5140
+    },
+    {
+      "epoch": 2.753637732062218,
+      "grad_norm": 11.940224560512375,
+      "learning_rate": 2.030140681910508e-08,
+      "logits/chosen": -0.22967776656150818,
+      "logits/rejected": -0.07449609041213989,
+      "logps/chosen": -1.2044155597686768,
+      "logps/rejected": -1.4070556163787842,
+      "loss": 1.5614,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.2044155597686768,
+      "rewards/margins": 0.20264002680778503,
+      "rewards/rejected": -1.4070556163787842,
+      "step": 5145
+    },
+    {
+      "epoch": 2.756313764843619,
+      "grad_norm": 5.597851916698418,
+      "learning_rate": 1.986445577483753e-08,
+      "logits/chosen": -0.2449430227279663,
+      "logits/rejected": -0.14069703221321106,
+      "logps/chosen": -1.1979897022247314,
+      "logps/rejected": -1.4494991302490234,
+      "loss": 1.552,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.1979897022247314,
+      "rewards/margins": 0.25150924921035767,
+      "rewards/rejected": -1.4494991302490234,
+      "step": 5150
+    },
+    {
+      "epoch": 2.758989797625021,
+      "grad_norm": 7.59879161071765,
+      "learning_rate": 1.9432163007725765e-08,
+      "logits/chosen": -0.28897973895072937,
+      "logits/rejected": -0.20519177615642548,
+      "logps/chosen": -1.268553376197815,
+      "logps/rejected": -1.505704402923584,
+      "loss": 1.5986,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.268553376197815,
+      "rewards/margins": 0.23715105652809143,
+      "rewards/rejected": -1.505704402923584,
+      "step": 5155
+    },
+    {
+      "epoch": 2.7616658304064226,
+      "grad_norm": 9.142398559940261,
+      "learning_rate": 1.9004532711876297e-08,
+      "logits/chosen": -0.2665271461009979,
+      "logits/rejected": -0.2148573398590088,
+      "logps/chosen": -1.186708927154541,
+      "logps/rejected": -1.5509085655212402,
+      "loss": 1.5012,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -1.186708927154541,
+      "rewards/margins": 0.364199697971344,
+      "rewards/rejected": -1.5509085655212402,
+      "step": 5160
+    },
+    {
+      "epoch": 2.7643418631878243,
+      "grad_norm": 8.065384544004745,
+      "learning_rate": 1.8581569036159928e-08,
+      "logits/chosen": -0.22522492706775665,
+      "logits/rejected": -0.06659802049398422,
+      "logps/chosen": -1.1889328956604004,
+      "logps/rejected": -1.4411547183990479,
+      "loss": 1.531,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.1889328956604004,
+      "rewards/margins": 0.2522217631340027,
+      "rewards/rejected": -1.4411547183990479,
+      "step": 5165
+    },
+    {
+      "epoch": 2.7670178959692255,
+      "grad_norm": 8.878700812965635,
+      "learning_rate": 1.8163276084172285e-08,
+      "logits/chosen": -0.1980402022600174,
+      "logits/rejected": -0.09475599229335785,
+      "logps/chosen": -1.2845584154129028,
+      "logps/rejected": -1.4850971698760986,
+      "loss": 1.6296,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.2845584154129028,
+      "rewards/margins": 0.2005387544631958,
+      "rewards/rejected": -1.4850971698760986,
+      "step": 5170
+    },
+    {
+      "epoch": 2.7696939287506273,
+      "grad_norm": 8.518784025504653,
+      "learning_rate": 1.7749657914193194e-08,
+      "logits/chosen": -0.254823237657547,
+      "logits/rejected": -0.16935142874717712,
+      "logps/chosen": -1.2913539409637451,
+      "logps/rejected": -1.4431110620498657,
+      "loss": 1.6446,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.2913539409637451,
+      "rewards/margins": 0.15175721049308777,
+      "rewards/rejected": -1.4431110620498657,
+      "step": 5175
+    },
+    {
+      "epoch": 2.7723699615320285,
+      "grad_norm": 6.991399757164978,
+      "learning_rate": 1.7340718539148203e-08,
+      "logits/chosen": -0.15637443959712982,
+      "logits/rejected": -0.12771818041801453,
+      "logps/chosen": -1.2784687280654907,
+      "logps/rejected": -1.5572850704193115,
+      "loss": 1.5896,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.2784687280654907,
+      "rewards/margins": 0.27881646156311035,
+      "rewards/rejected": -1.5572850704193115,
+      "step": 5180
+    },
+    {
+      "epoch": 2.7750459943134302,
+      "grad_norm": 12.336643207237364,
+      "learning_rate": 1.6936461926568724e-08,
+      "logits/chosen": -0.1913735717535019,
+      "logits/rejected": -0.09848286211490631,
+      "logps/chosen": -1.137644648551941,
+      "logps/rejected": -1.4215614795684814,
+      "loss": 1.5127,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.137644648551941,
+      "rewards/margins": 0.2839166224002838,
+      "rewards/rejected": -1.4215614795684814,
+      "step": 5185
+    },
+    {
+      "epoch": 2.777722027094832,
+      "grad_norm": 8.694579253035036,
+      "learning_rate": 1.6536891998554346e-08,
+      "logits/chosen": -0.29581698775291443,
+      "logits/rejected": -0.16581565141677856,
+      "logps/chosen": -1.2244768142700195,
+      "logps/rejected": -1.4132734537124634,
+      "loss": 1.566,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.2244768142700195,
+      "rewards/margins": 0.18879663944244385,
+      "rewards/rejected": -1.4132734537124634,
+      "step": 5190
+    },
+    {
+      "epoch": 2.7803980598762337,
+      "grad_norm": 12.053115358743863,
+      "learning_rate": 1.6142012631734093e-08,
+      "logits/chosen": -0.19457034766674042,
+      "logits/rejected": -0.0800018310546875,
+      "logps/chosen": -1.228502631187439,
+      "logps/rejected": -1.4483073949813843,
+      "loss": 1.5805,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.228502631187439,
+      "rewards/margins": 0.21980471909046173,
+      "rewards/rejected": -1.4483073949813843,
+      "step": 5195
+    },
+    {
+      "epoch": 2.783074092657635,
+      "grad_norm": 9.889001908094752,
+      "learning_rate": 1.575182765722949e-08,
+      "logits/chosen": -0.27219057083129883,
+      "logits/rejected": -0.15498098731040955,
+      "logps/chosen": -1.175596833229065,
+      "logps/rejected": -1.422145128250122,
+      "loss": 1.5671,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.175596833229065,
+      "rewards/margins": 0.24654832482337952,
+      "rewards/rejected": -1.422145128250122,
+      "step": 5200
+    },
+    {
+      "epoch": 2.783074092657635,
+      "eval_logits/chosen": 0.12440230697393417,
+      "eval_logits/rejected": 0.20024247467517853,
+      "eval_logps/chosen": -1.3029695749282837,
+      "eval_logps/rejected": -1.499582052230835,
+      "eval_loss": 1.6474473476409912,
+      "eval_rewards/accuracies": 0.5704748034477234,
+      "eval_rewards/chosen": -1.3029695749282837,
+      "eval_rewards/margins": 0.19661255180835724,
+      "eval_rewards/rejected": -1.499582052230835,
+      "eval_runtime": 40.378,
+      "eval_samples_per_second": 33.31,
+      "eval_steps_per_second": 8.346,
+      "step": 5200
+    },
+    {
+      "epoch": 2.7857501254390367,
+      "grad_norm": 7.894284148032132,
+      "learning_rate": 1.536634086061672e-08,
+      "logits/chosen": -0.11715634167194366,
+      "logits/rejected": -0.09793325513601303,
+      "logps/chosen": -1.2184431552886963,
+      "logps/rejected": -1.4105503559112549,
+      "loss": 1.5915,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -1.2184431552886963,
+      "rewards/margins": 0.19210708141326904,
+      "rewards/rejected": -1.4105503559112549,
+      "step": 5205
+    },
+    {
+      "epoch": 2.788426158220438,
+      "grad_norm": 8.626425862347766,
+      "learning_rate": 1.4985555981890495e-08,
+      "logits/chosen": -0.1889527142047882,
+      "logits/rejected": -0.12425367534160614,
+      "logps/chosen": -1.2231550216674805,
+      "logps/rejected": -1.4383323192596436,
+      "loss": 1.5682,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.2231550216674805,
+      "rewards/margins": 0.21517720818519592,
+      "rewards/rejected": -1.4383323192596436,
+      "step": 5210
+    },
+    {
+      "epoch": 2.7911021910018396,
+      "grad_norm": 7.747896620645018,
+      "learning_rate": 1.4609476715427226e-08,
+      "logits/chosen": -0.2118125855922699,
+      "logits/rejected": -0.13611485064029694,
+      "logps/chosen": -1.2079906463623047,
+      "logps/rejected": -1.5039584636688232,
+      "loss": 1.5452,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.2079906463623047,
+      "rewards/margins": 0.2959679663181305,
+      "rewards/rejected": -1.5039584636688232,
+      "step": 5215
+    },
+    {
+      "epoch": 2.7937782237832414,
+      "grad_norm": 10.303972437834066,
+      "learning_rate": 1.4238106709949792e-08,
+      "logits/chosen": -0.22145023941993713,
+      "logits/rejected": -0.1696382611989975,
+      "logps/chosen": -1.198301911354065,
+      "logps/rejected": -1.5057936906814575,
+      "loss": 1.5284,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.198301911354065,
+      "rewards/margins": 0.3074919283390045,
+      "rewards/rejected": -1.5057936906814575,
+      "step": 5220
+    },
+    {
+      "epoch": 2.796454256564643,
+      "grad_norm": 9.121670917332018,
+      "learning_rate": 1.3871449568491511e-08,
+      "logits/chosen": -0.16502538323402405,
+      "logits/rejected": -0.062241099774837494,
+      "logps/chosen": -1.2805616855621338,
+      "logps/rejected": -1.5200989246368408,
+      "loss": 1.5943,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.2805616855621338,
+      "rewards/margins": 0.2395373284816742,
+      "rewards/rejected": -1.5200989246368408,
+      "step": 5225
+    },
+    {
+      "epoch": 2.7991302893460444,
+      "grad_norm": 9.755347750909337,
+      "learning_rate": 1.3509508848361606e-08,
+      "logits/chosen": -0.31038230657577515,
+      "logits/rejected": -0.20442113280296326,
+      "logps/chosen": -1.2425405979156494,
+      "logps/rejected": -1.4383920431137085,
+      "loss": 1.59,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.2425405979156494,
+      "rewards/margins": 0.19585171341896057,
+      "rewards/rejected": -1.4383920431137085,
+      "step": 5230
+    },
+    {
+      "epoch": 2.801806322127446,
+      "grad_norm": 6.164015248610251,
+      "learning_rate": 1.3152288061110517e-08,
+      "logits/chosen": -0.26571184396743774,
+      "logits/rejected": -0.19366225600242615,
+      "logps/chosen": -1.251062035560608,
+      "logps/rejected": -1.454511284828186,
+      "loss": 1.5877,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.251062035560608,
+      "rewards/margins": 0.20344917476177216,
+      "rewards/rejected": -1.454511284828186,
+      "step": 5235
+    },
+    {
+      "epoch": 2.804482354908848,
+      "grad_norm": 9.239101149943929,
+      "learning_rate": 1.2799790672495814e-08,
+      "logits/chosen": -0.23363038897514343,
+      "logits/rejected": -0.05663818120956421,
+      "logps/chosen": -1.243467092514038,
+      "logps/rejected": -1.462083101272583,
+      "loss": 1.6168,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.243467092514038,
+      "rewards/margins": 0.2186162769794464,
+      "rewards/rejected": -1.462083101272583,
+      "step": 5240
+    },
+    {
+      "epoch": 2.807158387690249,
+      "grad_norm": 6.829649870494868,
+      "learning_rate": 1.2452020102448835e-08,
+      "logits/chosen": -0.13945211470127106,
+      "logits/rejected": -0.10049142688512802,
+      "logps/chosen": -1.194481372833252,
+      "logps/rejected": -1.3906782865524292,
+      "loss": 1.5771,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.194481372833252,
+      "rewards/margins": 0.1961970031261444,
+      "rewards/rejected": -1.3906782865524292,
+      "step": 5245
+    },
+    {
+      "epoch": 2.8098344204716508,
+      "grad_norm": 7.432169595362053,
+      "learning_rate": 1.2108979725041103e-08,
+      "logits/chosen": -0.23462161421775818,
+      "logits/rejected": -0.1215369701385498,
+      "logps/chosen": -1.270316481590271,
+      "logps/rejected": -1.473505973815918,
+      "loss": 1.6098,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.270316481590271,
+      "rewards/margins": 0.20318952202796936,
+      "rewards/rejected": -1.473505973815918,
+      "step": 5250
+    },
+    {
+      "epoch": 2.8125104532530525,
+      "grad_norm": 8.257547457316058,
+      "learning_rate": 1.1770672868451958e-08,
+      "logits/chosen": -0.1968560516834259,
+      "logits/rejected": -0.013767095282673836,
+      "logps/chosen": -1.2501423358917236,
+      "logps/rejected": -1.419557809829712,
+      "loss": 1.5999,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.2501423358917236,
+      "rewards/margins": 0.1694156527519226,
+      "rewards/rejected": -1.419557809829712,
+      "step": 5255
+    },
+    {
+      "epoch": 2.8151864860344538,
+      "grad_norm": 10.65316125130127,
+      "learning_rate": 1.1437102814935872e-08,
+      "logits/chosen": -0.2315382957458496,
+      "logits/rejected": -0.17422154545783997,
+      "logps/chosen": -1.2226743698120117,
+      "logps/rejected": -1.5226083993911743,
+      "loss": 1.58,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.2226743698120117,
+      "rewards/margins": 0.29993388056755066,
+      "rewards/rejected": -1.5226083993911743,
+      "step": 5260
+    },
+    {
+      "epoch": 2.8178625188158555,
+      "grad_norm": 8.085369009106172,
+      "learning_rate": 1.1108272800791018e-08,
+      "logits/chosen": -0.312086284160614,
+      "logits/rejected": -0.1401098519563675,
+      "logps/chosen": -1.405895709991455,
+      "logps/rejected": -1.5206575393676758,
+      "loss": 1.7369,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.405895709991455,
+      "rewards/margins": 0.11476186662912369,
+      "rewards/rejected": -1.5206575393676758,
+      "step": 5265
+    },
+    {
+      "epoch": 2.820538551597257,
+      "grad_norm": 6.393843195919607,
+      "learning_rate": 1.078418601632769e-08,
+      "logits/chosen": -0.19875682890415192,
+      "logits/rejected": -0.0823943018913269,
+      "logps/chosen": -1.1760179996490479,
+      "logps/rejected": -1.4837948083877563,
+      "loss": 1.5043,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.1760179996490479,
+      "rewards/margins": 0.3077767491340637,
+      "rewards/rejected": -1.4837948083877563,
+      "step": 5270
+    },
+    {
+      "epoch": 2.8232145843786585,
+      "grad_norm": 8.06384468638755,
+      "learning_rate": 1.0464845605837159e-08,
+      "logits/chosen": -0.20249497890472412,
+      "logits/rejected": -0.07072342932224274,
+      "logps/chosen": -1.254334807395935,
+      "logps/rejected": -1.4947919845581055,
+      "loss": 1.5855,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.254334807395935,
+      "rewards/margins": 0.24045705795288086,
+      "rewards/rejected": -1.4947919845581055,
+      "step": 5275
+    },
+    {
+      "epoch": 2.82589061716006,
+      "grad_norm": 9.025134402265097,
+      "learning_rate": 1.0150254667561642e-08,
+      "logits/chosen": -0.19225776195526123,
+      "logits/rejected": -0.060486339032649994,
+      "logps/chosen": -1.291597604751587,
+      "logps/rejected": -1.489046573638916,
+      "loss": 1.6364,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.291597604751587,
+      "rewards/margins": 0.19744889438152313,
+      "rewards/rejected": -1.489046573638916,
+      "step": 5280
+    },
+    {
+      "epoch": 2.828566649941462,
+      "grad_norm": 7.938475769626625,
+      "learning_rate": 9.840416253663719e-09,
+      "logits/chosen": -0.2338906079530716,
+      "logits/rejected": -0.1550368070602417,
+      "logps/chosen": -1.194307565689087,
+      "logps/rejected": -1.4473832845687866,
+      "loss": 1.5363,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.194307565689087,
+      "rewards/margins": 0.253075510263443,
+      "rewards/rejected": -1.4473832845687866,
+      "step": 5285
+    },
+    {
+      "epoch": 2.8312426827228636,
+      "grad_norm": 7.118025400739444,
+      "learning_rate": 9.535333370197074e-09,
+      "logits/chosen": -0.21142733097076416,
+      "logits/rejected": -0.0969664454460144,
+      "logps/chosen": -1.2613712549209595,
+      "logps/rejected": -1.4549555778503418,
+      "loss": 1.6146,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.2613712549209595,
+      "rewards/margins": 0.19358427822589874,
+      "rewards/rejected": -1.4549555778503418,
+      "step": 5290
+    },
+    {
+      "epoch": 2.833918715504265,
+      "grad_norm": 6.376414171175719,
+      "learning_rate": 9.23500897707713e-09,
+      "logits/chosen": -0.2588057518005371,
+      "logits/rejected": -0.10172237455844879,
+      "logps/chosen": -1.32537043094635,
+      "logps/rejected": -1.5747493505477905,
+      "loss": 1.6308,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.32537043094635,
+      "rewards/margins": 0.24937906861305237,
+      "rewards/rejected": -1.5747493505477905,
+      "step": 5295
+    },
+    {
+      "epoch": 2.8365947482856666,
+      "grad_norm": 7.875069396310308,
+      "learning_rate": 8.939445988052574e-09,
+      "logits/chosen": -0.20552654564380646,
+      "logits/rejected": -0.17443294823169708,
+      "logps/chosen": -1.2336318492889404,
+      "logps/rejected": -1.5586997270584106,
+      "loss": 1.5621,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.2336318492889404,
+      "rewards/margins": 0.3250678479671478,
+      "rewards/rejected": -1.5586997270584106,
+      "step": 5300
+    },
+    {
+      "epoch": 2.839270781067068,
+      "grad_norm": 8.252376839450696,
+      "learning_rate": 8.648647270676656e-09,
+      "logits/chosen": -0.18448421359062195,
+      "logits/rejected": -0.06468196958303452,
+      "logps/chosen": -1.285400152206421,
+      "logps/rejected": -1.5381015539169312,
+      "loss": 1.6323,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.285400152206421,
+      "rewards/margins": 0.25270140171051025,
+      "rewards/rejected": -1.5381015539169312,
+      "step": 5305
+    },
+    {
+      "epoch": 2.8419468138484696,
+      "grad_norm": 5.856217296203612,
+      "learning_rate": 8.362615646279991e-09,
+      "logits/chosen": -0.3467407822608948,
+      "logits/rejected": -0.13658985495567322,
+      "logps/chosen": -1.1790553331375122,
+      "logps/rejected": -1.4979755878448486,
+      "loss": 1.5298,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.1790553331375122,
+      "rewards/margins": 0.3189201354980469,
+      "rewards/rejected": -1.4979755878448486,
+      "step": 5310
+    },
+    {
+      "epoch": 2.8446228466298713,
+      "grad_norm": 9.410102772135225,
+      "learning_rate": 8.081353889942466e-09,
+      "logits/chosen": -0.13126492500305176,
+      "logits/rejected": -0.05035420134663582,
+      "logps/chosen": -1.2153863906860352,
+      "logps/rejected": -1.3907142877578735,
+      "loss": 1.5759,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.2153863906860352,
+      "rewards/margins": 0.175327867269516,
+      "rewards/rejected": -1.3907142877578735,
+      "step": 5315
+    },
+    {
+      "epoch": 2.847298879411273,
+      "grad_norm": 8.792712405679417,
+      "learning_rate": 7.804864730467042e-09,
+      "logits/chosen": -0.11626704037189484,
+      "logits/rejected": -0.05582579970359802,
+      "logps/chosen": -1.2154616117477417,
+      "logps/rejected": -1.3626997470855713,
+      "loss": 1.5908,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.2154616117477417,
+      "rewards/margins": 0.14723792672157288,
+      "rewards/rejected": -1.3626997470855713,
+      "step": 5320
+    },
+    {
+      "epoch": 2.8499749121926743,
+      "grad_norm": 7.961338709116618,
+      "learning_rate": 7.533150850352665e-09,
+      "logits/chosen": -0.1864510327577591,
+      "logits/rejected": -0.0834638699889183,
+      "logps/chosen": -1.262117862701416,
+      "logps/rejected": -1.5629605054855347,
+      "loss": 1.5815,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.262117862701416,
+      "rewards/margins": 0.30084237456321716,
+      "rewards/rejected": -1.5629605054855347,
+      "step": 5325
+    },
+    {
+      "epoch": 2.852650944974076,
+      "grad_norm": 9.561337144875273,
+      "learning_rate": 7.2662148857686175e-09,
+      "logits/chosen": -0.13462045788764954,
+      "logits/rejected": -0.08371740579605103,
+      "logps/chosen": -1.1950032711029053,
+      "logps/rejected": -1.4544382095336914,
+      "loss": 1.5301,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.1950032711029053,
+      "rewards/margins": 0.2594349682331085,
+      "rewards/rejected": -1.4544382095336914,
+      "step": 5330
+    },
+    {
+      "epoch": 2.8553269777554773,
+      "grad_norm": 13.869066735304004,
+      "learning_rate": 7.0040594265287635e-09,
+      "logits/chosen": -0.10755538940429688,
+      "logits/rejected": -0.13098764419555664,
+      "logps/chosen": -1.2394194602966309,
+      "logps/rejected": -1.4161888360977173,
+      "loss": 1.597,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.2394194602966309,
+      "rewards/margins": 0.17676934599876404,
+      "rewards/rejected": -1.4161888360977173,
+      "step": 5335
+    },
+    {
+      "epoch": 2.858003010536879,
+      "grad_norm": 8.208179342259479,
+      "learning_rate": 6.746687016066566e-09,
+      "logits/chosen": -0.14634302258491516,
+      "logits/rejected": -0.13666020333766937,
+      "logps/chosen": -1.258013129234314,
+      "logps/rejected": -1.4033854007720947,
+      "loss": 1.6307,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.258013129234314,
+      "rewards/margins": 0.1453721970319748,
+      "rewards/rejected": -1.4033854007720947,
+      "step": 5340
+    },
+    {
+      "epoch": 2.8606790433182807,
+      "grad_norm": 6.851778857771071,
+      "learning_rate": 6.494100151410276e-09,
+      "logits/chosen": -0.2954311966896057,
+      "logits/rejected": -0.1517902910709381,
+      "logps/chosen": -1.1930725574493408,
+      "logps/rejected": -1.4258384704589844,
+      "loss": 1.5191,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.1930725574493408,
+      "rewards/margins": 0.2327660322189331,
+      "rewards/rejected": -1.4258384704589844,
+      "step": 5345
+    },
+    {
+      "epoch": 2.8633550760996824,
+      "grad_norm": 10.521025339927364,
+      "learning_rate": 6.246301283158728e-09,
+      "logits/chosen": -0.1592158079147339,
+      "logits/rejected": -0.19058796763420105,
+      "logps/chosen": -1.3086292743682861,
+      "logps/rejected": -1.5055660009384155,
+      "loss": 1.6591,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.3086292743682861,
+      "rewards/margins": 0.1969369798898697,
+      "rewards/rejected": -1.5055660009384155,
+      "step": 5350
+    },
+    {
+      "epoch": 2.8660311088810837,
+      "grad_norm": 9.041618099717345,
+      "learning_rate": 6.0032928154576944e-09,
+      "logits/chosen": -0.22521352767944336,
+      "logits/rejected": -0.15095236897468567,
+      "logps/chosen": -1.2401598691940308,
+      "logps/rejected": -1.4540021419525146,
+      "loss": 1.6007,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.2401598691940308,
+      "rewards/margins": 0.21384210884571075,
+      "rewards/rejected": -1.4540021419525146,
+      "step": 5355
+    },
+    {
+      "epoch": 2.8687071416624854,
+      "grad_norm": 9.742275839581838,
+      "learning_rate": 5.76507710597629e-09,
+      "logits/chosen": -0.20284557342529297,
+      "logits/rejected": -0.045843563973903656,
+      "logps/chosen": -1.2402071952819824,
+      "logps/rejected": -1.4867188930511475,
+      "loss": 1.6002,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.2402071952819824,
+      "rewards/margins": 0.24651166796684265,
+      "rewards/rejected": -1.4867188930511475,
+      "step": 5360
+    },
+    {
+      "epoch": 2.8713831744438867,
+      "grad_norm": 7.235815472685509,
+      "learning_rate": 5.531656465884438e-09,
+      "logits/chosen": -0.2580950856208801,
+      "logits/rejected": -0.12209419161081314,
+      "logps/chosen": -1.2364237308502197,
+      "logps/rejected": -1.5134259462356567,
+      "loss": 1.5656,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.2364237308502197,
+      "rewards/margins": 0.2770020663738251,
+      "rewards/rejected": -1.5134259462356567,
+      "step": 5365
+    },
+    {
+      "epoch": 2.8740592072252884,
+      "grad_norm": 9.834877063853899,
+      "learning_rate": 5.303033159830217e-09,
+      "logits/chosen": -0.14675696194171906,
+      "logits/rejected": -0.1237391009926796,
+      "logps/chosen": -1.2326931953430176,
+      "logps/rejected": -1.3666555881500244,
+      "loss": 1.6062,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.2326931953430176,
+      "rewards/margins": 0.1339622437953949,
+      "rewards/rejected": -1.3666555881500244,
+      "step": 5370
+    },
+    {
+      "epoch": 2.87673524000669,
+      "grad_norm": 8.08309568080615,
+      "learning_rate": 5.079209405917939e-09,
+      "logits/chosen": -0.20791010558605194,
+      "logits/rejected": -0.12605224549770355,
+      "logps/chosen": -1.1827991008758545,
+      "logps/rejected": -1.5293145179748535,
+      "loss": 1.5095,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.1827991008758545,
+      "rewards/margins": 0.34651538729667664,
+      "rewards/rejected": -1.5293145179748535,
+      "step": 5375
+    },
+    {
+      "epoch": 2.879411272788092,
+      "grad_norm": 6.139482732449664,
+      "learning_rate": 4.860187375686664e-09,
+      "logits/chosen": -0.2154158651828766,
+      "logits/rejected": -0.040366265922784805,
+      "logps/chosen": -1.3586286306381226,
+      "logps/rejected": -1.548743724822998,
+      "loss": 1.6622,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.3586286306381226,
+      "rewards/margins": 0.19011513888835907,
+      "rewards/rejected": -1.548743724822998,
+      "step": 5380
+    },
+    {
+      "epoch": 2.882087305569493,
+      "grad_norm": 7.257550059938905,
+      "learning_rate": 4.64596919408905e-09,
+      "logits/chosen": -0.1530151516199112,
+      "logits/rejected": -0.09397026151418686,
+      "logps/chosen": -1.217520833015442,
+      "logps/rejected": -1.400549292564392,
+      "loss": 1.5824,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.217520833015442,
+      "rewards/margins": 0.18302853405475616,
+      "rewards/rejected": -1.400549292564392,
+      "step": 5385
+    },
+    {
+      "epoch": 2.884763338350895,
+      "grad_norm": 6.777289153854856,
+      "learning_rate": 4.436556939470814e-09,
+      "logits/chosen": -0.16746576130390167,
+      "logits/rejected": -0.07902796566486359,
+      "logps/chosen": -1.2627239227294922,
+      "logps/rejected": -1.4349424839019775,
+      "loss": 1.6059,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.2627239227294922,
+      "rewards/margins": 0.17221860587596893,
+      "rewards/rejected": -1.4349424839019775,
+      "step": 5390
+    },
+    {
+      "epoch": 2.887439371132296,
+      "grad_norm": 6.679859669428982,
+      "learning_rate": 4.23195264355064e-09,
+      "logits/chosen": -0.327754408121109,
+      "logits/rejected": -0.16632018983364105,
+      "logps/chosen": -1.1831705570220947,
+      "logps/rejected": -1.4538441896438599,
+      "loss": 1.5111,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.1831705570220947,
+      "rewards/margins": 0.2706735134124756,
+      "rewards/rejected": -1.4538441896438599,
+      "step": 5395
+    },
+    {
+      "epoch": 2.890115403913698,
+      "grad_norm": 7.606677602141569,
+      "learning_rate": 4.032158291400245e-09,
+      "logits/chosen": -0.2535339891910553,
+      "logits/rejected": -0.049510832875967026,
+      "logps/chosen": -1.2447563409805298,
+      "logps/rejected": -1.6381620168685913,
+      "loss": 1.5732,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.2447563409805298,
+      "rewards/margins": 0.3934057354927063,
+      "rewards/rejected": -1.6381620168685913,
+      "step": 5400
+    },
+    {
+      "epoch": 2.8927914366950995,
+      "grad_norm": 9.116747966192884,
+      "learning_rate": 3.837175821425398e-09,
+      "logits/chosen": -0.16068437695503235,
+      "logits/rejected": -0.13778674602508545,
+      "logps/chosen": -1.300722360610962,
+      "logps/rejected": -1.568656325340271,
+      "loss": 1.6196,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.300722360610962,
+      "rewards/margins": 0.2679339051246643,
+      "rewards/rejected": -1.568656325340271,
+      "step": 5405
+    },
+    {
+      "epoch": 2.8954674694765012,
+      "grad_norm": 7.7744056104123445,
+      "learning_rate": 3.6470071253467683e-09,
+      "logits/chosen": -0.1354900598526001,
+      "logits/rejected": -0.029869627207517624,
+      "logps/chosen": -1.2225590944290161,
+      "logps/rejected": -1.5437707901000977,
+      "loss": 1.5594,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.2225590944290161,
+      "rewards/margins": 0.321211576461792,
+      "rewards/rejected": -1.5437707901000977,
+      "step": 5410
+    },
+    {
+      "epoch": 2.8981435022579025,
+      "grad_norm": 8.490174611217292,
+      "learning_rate": 3.461654048181939e-09,
+      "logits/chosen": -0.2366805523633957,
+      "logits/rejected": -0.10289554297924042,
+      "logps/chosen": -1.252111554145813,
+      "logps/rejected": -1.4506523609161377,
+      "loss": 1.5945,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.252111554145813,
+      "rewards/margins": 0.1985408365726471,
+      "rewards/rejected": -1.4506523609161377,
+      "step": 5415
+    },
+    {
+      "epoch": 2.9008195350393042,
+      "grad_norm": 9.58980198103724,
+      "learning_rate": 3.281118388227255e-09,
+      "logits/chosen": -0.17817652225494385,
+      "logits/rejected": -0.1203083023428917,
+      "logps/chosen": -1.1674425601959229,
+      "logps/rejected": -1.3943111896514893,
+      "loss": 1.541,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.1674425601959229,
+      "rewards/margins": 0.2268688678741455,
+      "rewards/rejected": -1.3943111896514893,
+      "step": 5420
+    },
+    {
+      "epoch": 2.903495567820706,
+      "grad_norm": 13.051780801828894,
+      "learning_rate": 3.1054018970405048e-09,
+      "logits/chosen": -0.18536776304244995,
+      "logits/rejected": -0.08422654867172241,
+      "logps/chosen": -1.272070288658142,
+      "logps/rejected": -1.519740343093872,
+      "loss": 1.6125,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.272070288658142,
+      "rewards/margins": 0.24767009913921356,
+      "rewards/rejected": -1.519740343093872,
+      "step": 5425
+    },
+    {
+      "epoch": 2.906171600602107,
+      "grad_norm": 9.637390983771256,
+      "learning_rate": 2.9345062794238207e-09,
+      "logits/chosen": -0.19817817211151123,
+      "logits/rejected": -0.05495418980717659,
+      "logps/chosen": -1.2120387554168701,
+      "logps/rejected": -1.5326441526412964,
+      "loss": 1.5384,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.2120387554168701,
+      "rewards/margins": 0.3206053376197815,
+      "rewards/rejected": -1.5326441526412964,
+      "step": 5430
+    },
+    {
+      "epoch": 2.908847633383509,
+      "grad_norm": 12.442063962181845,
+      "learning_rate": 2.7684331934072492e-09,
+      "logits/chosen": -0.3037939667701721,
+      "logits/rejected": -0.21871428191661835,
+      "logps/chosen": -1.2116082906723022,
+      "logps/rejected": -1.531022310256958,
+      "loss": 1.5294,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.2116082906723022,
+      "rewards/margins": 0.31941401958465576,
+      "rewards/rejected": -1.531022310256958,
+      "step": 5435
+    },
+    {
+      "epoch": 2.9115236661649107,
+      "grad_norm": 8.255143974951732,
+      "learning_rate": 2.6071842502326526e-09,
+      "logits/chosen": -0.2509545385837555,
+      "logits/rejected": -0.16142788529396057,
+      "logps/chosen": -1.2278988361358643,
+      "logps/rejected": -1.5453009605407715,
+      "loss": 1.5674,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -1.2278988361358643,
+      "rewards/margins": 0.3174021244049072,
+      "rewards/rejected": -1.5453009605407715,
+      "step": 5440
+    },
+    {
+      "epoch": 2.9141996989463124,
+      "grad_norm": 7.084364648224411,
+      "learning_rate": 2.450761014337888e-09,
+      "logits/chosen": -0.03262624889612198,
+      "logits/rejected": -0.03374551981687546,
+      "logps/chosen": -1.2018756866455078,
+      "logps/rejected": -1.586195707321167,
+      "loss": 1.5345,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.2018756866455078,
+      "rewards/margins": 0.3843199908733368,
+      "rewards/rejected": -1.586195707321167,
+      "step": 5445
+    },
+    {
+      "epoch": 2.9168757317277136,
+      "grad_norm": 9.392045491922225,
+      "learning_rate": 2.299165003341985e-09,
+      "logits/chosen": -0.13462188839912415,
+      "logits/rejected": -0.04474419727921486,
+      "logps/chosen": -1.2435557842254639,
+      "logps/rejected": -1.5604596138000488,
+      "loss": 1.5655,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.2435557842254639,
+      "rewards/margins": 0.31690365076065063,
+      "rewards/rejected": -1.5604596138000488,
+      "step": 5450
+    },
+    {
+      "epoch": 2.9195517645091154,
+      "grad_norm": 7.59640684513388,
+      "learning_rate": 2.1523976880299945e-09,
+      "logits/chosen": -0.2498139888048172,
+      "logits/rejected": -0.11051896959543228,
+      "logps/chosen": -1.2221424579620361,
+      "logps/rejected": -1.3954873085021973,
+      "loss": 1.5822,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.2221424579620361,
+      "rewards/margins": 0.17334489524364471,
+      "rewards/rejected": -1.3954873085021973,
+      "step": 5455
+    },
+    {
+      "epoch": 2.9222277972905166,
+      "grad_norm": 8.232145092021927,
+      "learning_rate": 2.010460492339161e-09,
+      "logits/chosen": -0.24411647021770477,
+      "logits/rejected": -0.17508909106254578,
+      "logps/chosen": -1.2442398071289062,
+      "logps/rejected": -1.5127445459365845,
+      "loss": 1.5682,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.2442398071289062,
+      "rewards/margins": 0.26850467920303345,
+      "rewards/rejected": -1.5127445459365845,
+      "step": 5460
+    },
+    {
+      "epoch": 2.9249038300719183,
+      "grad_norm": 8.596987983047574,
+      "learning_rate": 1.8733547933446614e-09,
+      "logits/chosen": -0.2940111756324768,
+      "logits/rejected": -0.14695706963539124,
+      "logps/chosen": -1.29569411277771,
+      "logps/rejected": -1.4681915044784546,
+      "loss": 1.643,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.29569411277771,
+      "rewards/margins": 0.17249757051467896,
+      "rewards/rejected": -1.4681915044784546,
+      "step": 5465
+    },
+    {
+      "epoch": 2.92757986285332,
+      "grad_norm": 10.207032165158024,
+      "learning_rate": 1.7410819212467231e-09,
+      "logits/chosen": -0.19064298272132874,
+      "logits/rejected": -0.12294765561819077,
+      "logps/chosen": -1.1765105724334717,
+      "logps/rejected": -1.425095796585083,
+      "loss": 1.5242,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.1765105724334717,
+      "rewards/margins": 0.2485850751399994,
+      "rewards/rejected": -1.425095796585083,
+      "step": 5470
+    },
+    {
+      "epoch": 2.9302558956347218,
+      "grad_norm": 8.050551279955041,
+      "learning_rate": 1.613643159357192e-09,
+      "logits/chosen": -0.14887556433677673,
+      "logits/rejected": -0.1944231539964676,
+      "logps/chosen": -1.1479804515838623,
+      "logps/rejected": -1.3909088373184204,
+      "loss": 1.5207,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.1479804515838623,
+      "rewards/margins": 0.2429281771183014,
+      "rewards/rejected": -1.3909088373184204,
+      "step": 5475
+    },
+    {
+      "epoch": 2.932931928416123,
+      "grad_norm": 7.206610777472823,
+      "learning_rate": 1.4910397440875967e-09,
+      "logits/chosen": -0.1818329095840454,
+      "logits/rejected": -0.0986550822854042,
+      "logps/chosen": -1.241981863975525,
+      "logps/rejected": -1.4175071716308594,
+      "loss": 1.6009,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.241981863975525,
+      "rewards/margins": 0.17552533745765686,
+      "rewards/rejected": -1.4175071716308594,
+      "step": 5480
+    },
+    {
+      "epoch": 2.9356079611975248,
+      "grad_norm": 8.150550298484777,
+      "learning_rate": 1.3732728649368253e-09,
+      "logits/chosen": -0.12400376796722412,
+      "logits/rejected": 0.01435978151857853,
+      "logps/chosen": -1.219214916229248,
+      "logps/rejected": -1.4300227165222168,
+      "loss": 1.5769,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.219214916229248,
+      "rewards/margins": 0.21080783009529114,
+      "rewards/rejected": -1.4300227165222168,
+      "step": 5485
+    },
+    {
+      "epoch": 2.938283993978926,
+      "grad_norm": 9.444641265654619,
+      "learning_rate": 1.260343664479524e-09,
+      "logits/chosen": -0.18277186155319214,
+      "logits/rejected": -0.15881676971912384,
+      "logps/chosen": -1.2164661884307861,
+      "logps/rejected": -1.3838369846343994,
+      "loss": 1.5771,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.2164661884307861,
+      "rewards/margins": 0.1673709601163864,
+      "rewards/rejected": -1.3838369846343994,
+      "step": 5490
+    },
+    {
+      "epoch": 2.9409600267603278,
+      "grad_norm": 7.254483347505405,
+      "learning_rate": 1.1522532383554384e-09,
+      "logits/chosen": -0.28208598494529724,
+      "logits/rejected": -0.11577792465686798,
+      "logps/chosen": -1.179441213607788,
+      "logps/rejected": -1.4718883037567139,
+      "loss": 1.5135,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.179441213607788,
+      "rewards/margins": 0.2924470007419586,
+      "rewards/rejected": -1.4718883037567139,
+      "step": 5495
+    },
+    {
+      "epoch": 2.9436360595417295,
+      "grad_norm": 7.158672078935296,
+      "learning_rate": 1.049002635258256e-09,
+      "logits/chosen": -0.2083779126405716,
+      "logits/rejected": -0.11806578934192657,
+      "logps/chosen": -1.2659683227539062,
+      "logps/rejected": -1.39592707157135,
+      "loss": 1.6229,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.2659683227539062,
+      "rewards/margins": 0.12995865941047668,
+      "rewards/rejected": -1.39592707157135,
+      "step": 5500
+    },
+    {
+      "epoch": 2.946312092323131,
+      "grad_norm": 6.341158898811177,
+      "learning_rate": 9.505928569258358e-10,
+      "logits/chosen": -0.12015247344970703,
+      "logits/rejected": -0.12586413323879242,
+      "logps/chosen": -1.2438194751739502,
+      "logps/rejected": -1.4896745681762695,
+      "loss": 1.5735,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.2438194751739502,
+      "rewards/margins": 0.2458551824092865,
+      "rewards/rejected": -1.4896745681762695,
+      "step": 5505
+    },
+    {
+      "epoch": 2.9489881251045325,
+      "grad_norm": 9.149303002417405,
+      "learning_rate": 8.57024858130273e-10,
+      "logits/chosen": -0.23201246559619904,
+      "logits/rejected": -0.15525290369987488,
+      "logps/chosen": -1.2421350479125977,
+      "logps/rejected": -1.585129737854004,
+      "loss": 1.5495,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.2421350479125977,
+      "rewards/margins": 0.3429946005344391,
+      "rewards/rejected": -1.585129737854004,
+      "step": 5510
+    },
+    {
+      "epoch": 2.951664157885934,
+      "grad_norm": 9.70660458120073,
+      "learning_rate": 7.682995466686826e-10,
+      "logits/chosen": -0.28649234771728516,
+      "logits/rejected": -0.1822827011346817,
+      "logps/chosen": -1.207561731338501,
+      "logps/rejected": -1.5450680255889893,
+      "loss": 1.5349,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -1.207561731338501,
+      "rewards/margins": 0.3375062346458435,
+      "rewards/rejected": -1.5450680255889893,
+      "step": 5515
+    },
+    {
+      "epoch": 2.9543401906673354,
+      "grad_norm": 9.893690419887266,
+      "learning_rate": 6.844177833543741e-10,
+      "logits/chosen": -0.1666533648967743,
+      "logits/rejected": -0.12191154062747955,
+      "logps/chosen": -1.1799771785736084,
+      "logps/rejected": -1.418250322341919,
+      "loss": 1.5562,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -1.1799771785736084,
+      "rewards/margins": 0.23827309906482697,
+      "rewards/rejected": -1.418250322341919,
+      "step": 5520
+    },
+    {
+      "epoch": 2.957016223448737,
+      "grad_norm": 6.773672161944208,
+      "learning_rate": 6.053803820087467e-10,
+      "logits/chosen": -0.18774524331092834,
+      "logits/rejected": -0.08795052021741867,
+      "logps/chosen": -1.2257516384124756,
+      "logps/rejected": -1.5606324672698975,
+      "loss": 1.5534,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -1.2257516384124756,
+      "rewards/margins": 0.33488088846206665,
+      "rewards/rejected": -1.5606324672698975,
+      "step": 5525
+    },
+    {
+      "epoch": 2.959692256230139,
+      "grad_norm": 12.468963843370322,
+      "learning_rate": 5.311881094528514e-10,
+      "logits/chosen": -0.21507851779460907,
+      "logits/rejected": -0.02370876632630825,
+      "logps/chosen": -1.3210976123809814,
+      "logps/rejected": -1.4538319110870361,
+      "loss": 1.6688,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.3210976123809814,
+      "rewards/margins": 0.13273414969444275,
+      "rewards/rejected": -1.4538319110870361,
+      "step": 5530
+    },
+    {
+      "epoch": 2.9623682890115406,
+      "grad_norm": 11.22212558598113,
+      "learning_rate": 4.6184168550050806e-10,
+      "logits/chosen": -0.2066337615251541,
+      "logits/rejected": -0.17999473214149475,
+      "logps/chosen": -1.2044970989227295,
+      "logps/rejected": -1.3566124439239502,
+      "loss": 1.5863,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.2044970989227295,
+      "rewards/margins": 0.1521153599023819,
+      "rewards/rejected": -1.3566124439239502,
+      "step": 5535
+    },
+    {
+      "epoch": 2.965044321792942,
+      "grad_norm": 9.236625587079201,
+      "learning_rate": 3.973417829510328e-10,
+      "logits/chosen": -0.3020160496234894,
+      "logits/rejected": -0.16865511238574982,
+      "logps/chosen": -1.278668999671936,
+      "logps/rejected": -1.4230049848556519,
+      "loss": 1.6192,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.278668999671936,
+      "rewards/margins": 0.1443360149860382,
+      "rewards/rejected": -1.4230049848556519,
+      "step": 5540
+    },
+    {
+      "epoch": 2.9677203545743436,
+      "grad_norm": 7.604616929280045,
+      "learning_rate": 3.3768902758274377e-10,
+      "logits/chosen": -0.18155820667743683,
+      "logits/rejected": -0.09487228095531464,
+      "logps/chosen": -1.1723692417144775,
+      "logps/rejected": -1.377142310142517,
+      "loss": 1.5527,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.1723692417144775,
+      "rewards/margins": 0.20477309823036194,
+      "rewards/rejected": -1.377142310142517,
+      "step": 5545
+    },
+    {
+      "epoch": 2.970396387355745,
+      "grad_norm": 7.8461369749242715,
+      "learning_rate": 2.8288399814691e-10,
+      "logits/chosen": -0.10973820835351944,
+      "logits/rejected": -0.012431502342224121,
+      "logps/chosen": -1.296041488647461,
+      "logps/rejected": -1.5153841972351074,
+      "loss": 1.6154,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.296041488647461,
+      "rewards/margins": 0.21934261918067932,
+      "rewards/rejected": -1.5153841972351074,
+      "step": 5550
+    },
+    {
+      "epoch": 2.9730724201371466,
+      "grad_norm": 9.744055832864698,
+      "learning_rate": 2.3292722636220066e-10,
+      "logits/chosen": -0.21309657394886017,
+      "logits/rejected": -0.040102262049913406,
+      "logps/chosen": -1.3197863101959229,
+      "logps/rejected": -1.5416650772094727,
+      "loss": 1.6391,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.3197863101959229,
+      "rewards/margins": 0.22187869250774384,
+      "rewards/rejected": -1.5416650772094727,
+      "step": 5555
+    },
+    {
+      "epoch": 2.9757484529185483,
+      "grad_norm": 7.960303028084376,
+      "learning_rate": 1.8781919690946668e-10,
+      "logits/chosen": -0.1571412831544876,
+      "logits/rejected": -0.15997639298439026,
+      "logps/chosen": -1.2541515827178955,
+      "logps/rejected": -1.3549838066101074,
+      "loss": 1.62,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.2541515827178955,
+      "rewards/margins": 0.10083191096782684,
+      "rewards/rejected": -1.3549838066101074,
+      "step": 5560
+    },
+    {
+      "epoch": 2.97842448569995,
+      "grad_norm": 11.345542567123752,
+      "learning_rate": 1.4756034742696711e-10,
+      "logits/chosen": -0.249413400888443,
+      "logits/rejected": -0.22129006683826447,
+      "logps/chosen": -1.2185876369476318,
+      "logps/rejected": -1.3922250270843506,
+      "loss": 1.5827,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.2185876369476318,
+      "rewards/margins": 0.17363746464252472,
+      "rewards/rejected": -1.3922250270843506,
+      "step": 5565
+    },
+    {
+      "epoch": 2.9811005184813513,
+      "grad_norm": 9.709369528026375,
+      "learning_rate": 1.12151068506261e-10,
+      "logits/chosen": -0.16392943263053894,
+      "logits/rejected": -0.06352823972702026,
+      "logps/chosen": -1.1880154609680176,
+      "logps/rejected": -1.525194764137268,
+      "loss": 1.5293,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.1880154609680176,
+      "rewards/margins": 0.3371793329715729,
+      "rewards/rejected": -1.525194764137268,
+      "step": 5570
+    },
+    {
+      "epoch": 2.983776551262753,
+      "grad_norm": 9.717767351843722,
+      "learning_rate": 8.159170368826629e-11,
+      "logits/chosen": -0.2174181491136551,
+      "logits/rejected": -0.09884576499462128,
+      "logps/chosen": -1.2130773067474365,
+      "logps/rejected": -1.4968039989471436,
+      "loss": 1.5547,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -1.2130773067474365,
+      "rewards/margins": 0.2837267816066742,
+      "rewards/rejected": -1.4968039989471436,
+      "step": 5575
+    },
+    {
+      "epoch": 2.9864525840441547,
+      "grad_norm": 8.268016813574327,
+      "learning_rate": 5.588254946015114e-11,
+      "logits/chosen": -0.2827422618865967,
+      "logits/rejected": -0.07571440190076828,
+      "logps/chosen": -1.1469650268554688,
+      "logps/rejected": -1.4948405027389526,
+      "loss": 1.5042,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -1.1469650268554688,
+      "rewards/margins": 0.3478756844997406,
+      "rewards/rejected": -1.4948405027389526,
+      "step": 5580
+    },
+    {
+      "epoch": 2.989128616825556,
+      "grad_norm": 9.069282097255376,
+      "learning_rate": 3.502385525216978e-11,
+      "logits/chosen": -0.2544441521167755,
+      "logits/rejected": -0.11987463384866714,
+      "logps/chosen": -1.2516387701034546,
+      "logps/rejected": -1.493402361869812,
+      "loss": 1.5747,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -1.2516387701034546,
+      "rewards/margins": 0.24176368117332458,
+      "rewards/rejected": -1.493402361869812,
+      "step": 5585
+    },
+    {
+      "epoch": 2.9918046496069577,
+      "grad_norm": 6.916859793628514,
+      "learning_rate": 1.901582343555308e-11,
+      "logits/chosen": -0.1889420449733734,
+      "logits/rejected": -0.15718932449817657,
+      "logps/chosen": -1.2972419261932373,
+      "logps/rejected": -1.4586095809936523,
+      "loss": 1.6364,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -1.2972419261932373,
+      "rewards/margins": 0.1613677740097046,
+      "rewards/rejected": -1.4586095809936523,
+      "step": 5590
+    },
+    {
+      "epoch": 2.9944806823883594,
+      "grad_norm": 9.315142123250766,
+      "learning_rate": 7.858609320232634e-12,
+      "logits/chosen": -0.20134368538856506,
+      "logits/rejected": -0.09880489856004715,
+      "logps/chosen": -1.1670615673065186,
+      "logps/rejected": -1.4292680025100708,
+      "loss": 1.5248,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -1.1670615673065186,
+      "rewards/margins": 0.262206494808197,
+      "rewards/rejected": -1.4292680025100708,
+      "step": 5595
+    },
+    {
+      "epoch": 2.9971567151697607,
+      "grad_norm": 9.052815076284157,
+      "learning_rate": 1.5523211535639624e-12,
+      "logits/chosen": -0.19403724372386932,
+      "logits/rejected": -0.12277773767709732,
+      "logps/chosen": -1.2054804563522339,
+      "logps/rejected": -1.5248081684112549,
+      "loss": 1.5485,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.2054804563522339,
+      "rewards/margins": 0.3193278908729553,
+      "rewards/rejected": -1.5248081684112549,
+      "step": 5600
+    },
+    {
+      "epoch": 2.9971567151697607,
+      "eval_logits/chosen": 0.07478887587785721,
+      "eval_logits/rejected": 0.14641378819942474,
+      "eval_logps/chosen": -1.3031429052352905,
+      "eval_logps/rejected": -1.499267339706421,
+      "eval_loss": 1.647899866104126,
+      "eval_rewards/accuracies": 0.5712166428565979,
+      "eval_rewards/chosen": -1.3031429052352905,
+      "eval_rewards/margins": 0.19612456858158112,
+      "eval_rewards/rejected": -1.499267339706421,
+      "eval_runtime": 40.482,
+      "eval_samples_per_second": 33.225,
+      "eval_steps_per_second": 8.325,
+      "step": 5600
+    },
+    {
+      "epoch": 2.999297541394882,
+      "step": 5604,
+      "total_flos": 0.0,
+      "train_loss": 1.6264377747324006,
+      "train_runtime": 30342.8146,
+      "train_samples_per_second": 5.911,
+      "train_steps_per_second": 0.185
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 5604,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 1000000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}