diff --git "a/trainer_state.json" "b/trainer_state.json"
deleted file mode 100644--- "a/trainer_state.json"
+++ /dev/null
@@ -1,4566 +0,0 @@
-{
-  "best_metric": null,
-  "best_model_checkpoint": null,
-  "epoch": 3.0,
-  "eval_steps": 100,
-  "global_step": 2907,
-  "is_hyper_param_search": false,
-  "is_local_process_zero": true,
-  "is_world_process_zero": true,
-  "log_history": [
-    {
-      "epoch": 0.0,
-      "learning_rate": 1.7182130584192438e-09,
-      "logits/chosen": -2.603872060775757,
-      "logits/rejected": -2.555414915084839,
-      "logps/chosen": -134.93458557128906,
-      "logps/rejected": -134.85955810546875,
-      "loss": 0.6931,
-      "rewards/accuracies": 0.0,
-      "rewards/chosen": 0.0,
-      "rewards/margins": 0.0,
-      "rewards/rejected": 0.0,
-      "step": 1
-    },
-    {
-      "epoch": 0.01,
-      "learning_rate": 1.718213058419244e-08,
-      "logits/chosen": -2.729396343231201,
-      "logits/rejected": -2.6624577045440674,
-      "logps/chosen": -303.16815185546875,
-      "logps/rejected": -252.68316650390625,
-      "loss": 0.6938,
-      "rewards/accuracies": 0.4722222089767456,
-      "rewards/chosen": 0.0021914299577474594,
-      "rewards/margins": 0.004278275184333324,
-      "rewards/rejected": -0.0020868456922471523,
-      "step": 10
-    },
-    {
-      "epoch": 0.02,
-      "learning_rate": 3.436426116838488e-08,
-      "logits/chosen": -2.747986078262329,
-      "logits/rejected": -2.6914308071136475,
-      "logps/chosen": -328.33062744140625,
-      "logps/rejected": -288.1149597167969,
-      "loss": 0.6912,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": -0.009702973067760468,
-      "rewards/margins": -0.0010181829566136003,
-      "rewards/rejected": -0.008684789761900902,
-      "step": 20
-    },
-    {
-      "epoch": 0.03,
-      "learning_rate": 5.154639175257731e-08,
-      "logits/chosen": -2.769310712814331,
-      "logits/rejected": -2.707155466079712,
-      "logps/chosen": -337.4163513183594,
-      "logps/rejected": -276.3641662597656,
-      "loss": 0.6925,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": -0.010683250613510609,
-      "rewards/margins": -0.006997206248342991,
-      "rewards/rejected": -0.0036860438995063305,
-      "step": 30
-    },
-    {
-      "epoch": 0.04,
-      "learning_rate": 6.872852233676976e-08,
-      "logits/chosen": -2.7873435020446777,
-      "logits/rejected": -2.7888503074645996,
-      "logps/chosen": -382.8026123046875,
-      "logps/rejected": -322.3409729003906,
-      "loss": 0.6934,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.0037022079341113567,
-      "rewards/margins": 0.011960670351982117,
-      "rewards/rejected": -0.008258461952209473,
-      "step": 40
-    },
-    {
-      "epoch": 0.05,
-      "learning_rate": 8.59106529209622e-08,
-      "logits/chosen": -2.660407781600952,
-      "logits/rejected": -2.6068549156188965,
-      "logps/chosen": -396.0019836425781,
-      "logps/rejected": -251.8150177001953,
-      "loss": 0.6911,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.004951278679072857,
-      "rewards/margins": 0.004014839418232441,
-      "rewards/rejected": 0.0009364400175400078,
-      "step": 50
-    },
-    {
-      "epoch": 0.06,
-      "learning_rate": 1.0309278350515462e-07,
-      "logits/chosen": -2.669360399246216,
-      "logits/rejected": -2.6709377765655518,
-      "logps/chosen": -312.83160400390625,
-      "logps/rejected": -266.94866943359375,
-      "loss": 0.6887,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.007568955421447754,
-      "rewards/margins": 0.01419537328183651,
-      "rewards/rejected": -0.0066264173947274685,
-      "step": 60
-    },
-    {
-      "epoch": 0.07,
-      "learning_rate": 1.202749140893471e-07,
-      "logits/chosen": -2.8407368659973145,
-      "logits/rejected": -2.717714309692383,
-      "logps/chosen": -418.32666015625,
-      "logps/rejected": -295.38348388671875,
-      "loss": 0.688,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.00732302526012063,
-      "rewards/margins": 0.0100878169760108,
-      "rewards/rejected": -0.0027647926472127438,
-      "step": 70
-    },
-    {
-      "epoch": 0.08,
-      "learning_rate": 1.3745704467353952e-07,
-      "logits/chosen": -2.7577030658721924,
-      "logits/rejected": -2.6890294551849365,
-      "logps/chosen": -340.7475891113281,
-      "logps/rejected": -262.4281311035156,
-      "loss": 0.6821,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.02396738901734352,
-      "rewards/margins": 0.032635144889354706,
-      "rewards/rejected": -0.00866775494068861,
-      "step": 80
-    },
-    {
-      "epoch": 0.09,
-      "learning_rate": 1.5463917525773197e-07,
-      "logits/chosen": -2.7611238956451416,
-      "logits/rejected": -2.721151828765869,
-      "logps/chosen": -333.66485595703125,
-      "logps/rejected": -255.80520629882812,
-      "loss": 0.6742,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.045851536095142365,
-      "rewards/margins": 0.06294064968824387,
-      "rewards/rejected": -0.017089109867811203,
-      "step": 90
-    },
-    {
-      "epoch": 0.1,
-      "learning_rate": 1.718213058419244e-07,
-      "logits/chosen": -2.646711826324463,
-      "logits/rejected": -2.6414952278137207,
-      "logps/chosen": -351.0623474121094,
-      "logps/rejected": -238.76431274414062,
-      "loss": 0.6686,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.06126219779253006,
-      "rewards/margins": 0.06746932864189148,
-      "rewards/rejected": -0.0062071289867162704,
-      "step": 100
-    },
-    {
-      "epoch": 0.1,
-      "eval_logits/chosen": -2.832993984222412,
-      "eval_logits/rejected": -2.7623188495635986,
-      "eval_logps/chosen": -369.2313232421875,
-      "eval_logps/rejected": -285.8178405761719,
-      "eval_loss": 0.6668010950088501,
-      "eval_rewards/accuracies": 0.6746031641960144,
-      "eval_rewards/chosen": 0.06240652874112129,
-      "eval_rewards/margins": 0.07296741008758545,
-      "eval_rewards/rejected": -0.010560884140431881,
-      "eval_runtime": 94.0983,
-      "eval_samples_per_second": 21.254,
-      "eval_steps_per_second": 0.67,
-      "step": 100
-    },
-    {
-      "epoch": 0.11,
-      "learning_rate": 1.8900343642611682e-07,
-      "logits/chosen": -2.738068103790283,
-      "logits/rejected": -2.659058094024658,
-      "logps/chosen": -306.91400146484375,
-      "logps/rejected": -279.0736083984375,
-      "loss": 0.6639,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.04395968094468117,
-      "rewards/margins": 0.057025860995054245,
-      "rewards/rejected": -0.01306617446243763,
-      "step": 110
-    },
-    {
-      "epoch": 0.12,
-      "learning_rate": 2.0618556701030925e-07,
-      "logits/chosen": -2.696042776107788,
-      "logits/rejected": -2.673532009124756,
-      "logps/chosen": -397.77996826171875,
-      "logps/rejected": -285.76092529296875,
-      "loss": 0.6523,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.08170491456985474,
-      "rewards/margins": 0.08871547877788544,
-      "rewards/rejected": -0.0070105730555951595,
-      "step": 120
-    },
-    {
-      "epoch": 0.13,
-      "learning_rate": 2.2336769759450173e-07,
-      "logits/chosen": -2.7733514308929443,
-      "logits/rejected": -2.7449357509613037,
-      "logps/chosen": -363.5209045410156,
-      "logps/rejected": -293.55010986328125,
-      "loss": 0.6474,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.08804867416620255,
-      "rewards/margins": 0.12048058211803436,
-      "rewards/rejected": -0.03243190422654152,
-      "step": 130
-    },
-    {
-      "epoch": 0.14,
-      "learning_rate": 2.405498281786942e-07,
-      "logits/chosen": -2.7430038452148438,
-      "logits/rejected": -2.6958515644073486,
-      "logps/chosen": -346.5725402832031,
-      "logps/rejected": -263.58282470703125,
-      "loss": 0.6365,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.14620031416416168,
-      "rewards/margins": 0.18137127161026,
-      "rewards/rejected": -0.03517094999551773,
-      "step": 140
-    },
-    {
-      "epoch": 0.15,
-      "learning_rate": 2.5773195876288655e-07,
-      "logits/chosen": -2.718210220336914,
-      "logits/rejected": -2.722475528717041,
-      "logps/chosen": -345.8394470214844,
-      "logps/rejected": -273.95721435546875,
-      "loss": 0.6426,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.09584580361843109,
-      "rewards/margins": 0.1124863252043724,
-      "rewards/rejected": -0.016640519723296165,
-      "step": 150
-    },
-    {
-      "epoch": 0.17,
-      "learning_rate": 2.7491408934707903e-07,
-      "logits/chosen": -2.86755633354187,
-      "logits/rejected": -2.796311855316162,
-      "logps/chosen": -387.51531982421875,
-      "logps/rejected": -289.4734802246094,
-      "loss": 0.6193,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.09817598015069962,
-      "rewards/margins": 0.20854738354682922,
-      "rewards/rejected": -0.11037138849496841,
-      "step": 160
-    },
-    {
-      "epoch": 0.18,
-      "learning_rate": 2.9209621993127146e-07,
-      "logits/chosen": -2.7607944011688232,
-      "logits/rejected": -2.7413511276245117,
-      "logps/chosen": -345.589599609375,
-      "logps/rejected": -276.03033447265625,
-      "loss": 0.6153,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.08899159729480743,
-      "rewards/margins": 0.13915829360485077,
-      "rewards/rejected": -0.05016670748591423,
-      "step": 170
-    },
-    {
-      "epoch": 0.19,
-      "learning_rate": 3.0927835051546394e-07,
-      "logits/chosen": -2.7164695262908936,
-      "logits/rejected": -2.703003406524658,
-      "logps/chosen": -275.18792724609375,
-      "logps/rejected": -236.8255615234375,
-      "loss": 0.6083,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.09344159066677094,
-      "rewards/margins": 0.23309943079948425,
-      "rewards/rejected": -0.1396578848361969,
-      "step": 180
-    },
-    {
-      "epoch": 0.2,
-      "learning_rate": 3.2646048109965636e-07,
-      "logits/chosen": -2.7676727771759033,
-      "logits/rejected": -2.69238018989563,
-      "logps/chosen": -346.99774169921875,
-      "logps/rejected": -237.373046875,
-      "loss": 0.6073,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.1757623851299286,
-      "rewards/margins": 0.30902284383773804,
-      "rewards/rejected": -0.13326042890548706,
-      "step": 190
-    },
-    {
-      "epoch": 0.21,
-      "learning_rate": 3.436426116838488e-07,
-      "logits/chosen": -2.6943726539611816,
-      "logits/rejected": -2.667006015777588,
-      "logps/chosen": -292.3045959472656,
-      "logps/rejected": -197.8688201904297,
-      "loss": 0.59,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.1381949484348297,
-      "rewards/margins": 0.3062794804573059,
-      "rewards/rejected": -0.1680845469236374,
-      "step": 200
-    },
-    {
-      "epoch": 0.21,
-      "eval_logits/chosen": -2.8180224895477295,
-      "eval_logits/rejected": -2.7513797283172607,
-      "eval_logps/chosen": -368.2522277832031,
-      "eval_logps/rejected": -287.6386413574219,
-      "eval_loss": 0.5995281934738159,
-      "eval_rewards/accuracies": 0.682539701461792,
-      "eval_rewards/chosen": 0.16031166911125183,
-      "eval_rewards/margins": 0.35295677185058594,
-      "eval_rewards/rejected": -0.1926451176404953,
-      "eval_runtime": 93.7328,
-      "eval_samples_per_second": 21.337,
-      "eval_steps_per_second": 0.672,
-      "step": 200
-    },
-    {
-      "epoch": 0.22,
-      "learning_rate": 3.608247422680412e-07,
-      "logits/chosen": -2.729691743850708,
-      "logits/rejected": -2.665153980255127,
-      "logps/chosen": -385.1280517578125,
-      "logps/rejected": -253.19482421875,
-      "loss": 0.5794,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.13179190456867218,
-      "rewards/margins": 0.3481922745704651,
-      "rewards/rejected": -0.2164003849029541,
-      "step": 210
-    },
-    {
-      "epoch": 0.23,
-      "learning_rate": 3.7800687285223364e-07,
-      "logits/chosen": -2.6447486877441406,
-      "logits/rejected": -2.5924179553985596,
-      "logps/chosen": -278.9236145019531,
-      "logps/rejected": -248.78488159179688,
-      "loss": 0.5758,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.10307928174734116,
-      "rewards/margins": 0.3263222277164459,
-      "rewards/rejected": -0.22324290871620178,
-      "step": 220
-    },
-    {
-      "epoch": 0.24,
-      "learning_rate": 3.9518900343642607e-07,
-      "logits/chosen": -2.627690553665161,
-      "logits/rejected": -2.6008286476135254,
-      "logps/chosen": -311.2276306152344,
-      "logps/rejected": -265.7589416503906,
-      "loss": 0.6094,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.10903775691986084,
-      "rewards/margins": 0.3014870285987854,
-      "rewards/rejected": -0.19244930148124695,
-      "step": 230
-    },
-    {
-      "epoch": 0.25,
-      "learning_rate": 4.123711340206185e-07,
-      "logits/chosen": -2.7416341304779053,
-      "logits/rejected": -2.696104049682617,
-      "logps/chosen": -388.8735046386719,
-      "logps/rejected": -255.23745727539062,
-      "loss": 0.5745,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.14113590121269226,
-      "rewards/margins": 0.35784265398979187,
-      "rewards/rejected": -0.21670672297477722,
-      "step": 240
-    },
-    {
-      "epoch": 0.26,
-      "learning_rate": 4.2955326460481097e-07,
-      "logits/chosen": -2.6901490688323975,
-      "logits/rejected": -2.6809473037719727,
-      "logps/chosen": -334.62066650390625,
-      "logps/rejected": -278.7101745605469,
-      "loss": 0.5903,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.13628730177879333,
-      "rewards/margins": 0.3466246426105499,
-      "rewards/rejected": -0.21033735573291779,
-      "step": 250
-    },
-    {
-      "epoch": 0.27,
-      "learning_rate": 4.4673539518900345e-07,
-      "logits/chosen": -2.7171425819396973,
-      "logits/rejected": -2.661024332046509,
-      "logps/chosen": -344.5673828125,
-      "logps/rejected": -245.82632446289062,
-      "loss": 0.6,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.0464027114212513,
-      "rewards/margins": 0.39084523916244507,
-      "rewards/rejected": -0.3444425165653229,
-      "step": 260
-    },
-    {
-      "epoch": 0.28,
-      "learning_rate": 4.639175257731959e-07,
-      "logits/chosen": -2.7654361724853516,
-      "logits/rejected": -2.7372591495513916,
-      "logps/chosen": -363.6313171386719,
-      "logps/rejected": -281.4266662597656,
-      "loss": 0.5987,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.012831795029342175,
-      "rewards/margins": 0.3564152419567108,
-      "rewards/rejected": -0.34358346462249756,
-      "step": 270
-    },
-    {
-      "epoch": 0.29,
-      "learning_rate": 4.810996563573884e-07,
-      "logits/chosen": -2.698306083679199,
-      "logits/rejected": -2.617525577545166,
-      "logps/chosen": -346.89898681640625,
-      "logps/rejected": -266.34173583984375,
-      "loss": 0.5745,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.1134389191865921,
-      "rewards/margins": 0.4053812026977539,
-      "rewards/rejected": -0.291942298412323,
-      "step": 280
-    },
-    {
-      "epoch": 0.3,
-      "learning_rate": 4.982817869415807e-07,
-      "logits/chosen": -2.7834925651550293,
-      "logits/rejected": -2.7064177989959717,
-      "logps/chosen": -352.7433166503906,
-      "logps/rejected": -262.48480224609375,
-      "loss": 0.5738,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.2624431550502777,
-      "rewards/margins": 0.535737156867981,
-      "rewards/rejected": -0.27329397201538086,
-      "step": 290
-    },
-    {
-      "epoch": 0.31,
-      "learning_rate": 4.982798165137615e-07,
-      "logits/chosen": -2.594956398010254,
-      "logits/rejected": -2.5584628582000732,
-      "logps/chosen": -290.9082946777344,
-      "logps/rejected": -268.2087707519531,
-      "loss": 0.5843,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.15836095809936523,
-      "rewards/margins": 0.3992231786251068,
-      "rewards/rejected": -0.24086228013038635,
-      "step": 300
-    },
-    {
-      "epoch": 0.31,
-      "eval_logits/chosen": -2.795222282409668,
-      "eval_logits/rejected": -2.7304515838623047,
-      "eval_logps/chosen": -367.5863952636719,
-      "eval_logps/rejected": -288.8868408203125,
-      "eval_loss": 0.5643562078475952,
-      "eval_rewards/accuracies": 0.6904761791229248,
-      "eval_rewards/chosen": 0.22689564526081085,
-      "eval_rewards/margins": 0.5443611145019531,
-      "eval_rewards/rejected": -0.3174654543399811,
-      "eval_runtime": 93.6627,
-      "eval_samples_per_second": 21.353,
-      "eval_steps_per_second": 0.673,
-      "step": 300
-    },
-    {
-      "epoch": 0.32,
-      "learning_rate": 4.963685015290519e-07,
-      "logits/chosen": -2.7573087215423584,
-      "logits/rejected": -2.7624802589416504,
-      "logps/chosen": -391.13470458984375,
-      "logps/rejected": -319.02850341796875,
-      "loss": 0.5823,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.17492187023162842,
-      "rewards/margins": 0.39395055174827576,
-      "rewards/rejected": -0.21902871131896973,
-      "step": 310
-    },
-    {
-      "epoch": 0.33,
-      "learning_rate": 4.944571865443424e-07,
-      "logits/chosen": -2.627657175064087,
-      "logits/rejected": -2.559328317642212,
-      "logps/chosen": -325.2025451660156,
-      "logps/rejected": -215.1479949951172,
-      "loss": 0.5405,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.17855343222618103,
-      "rewards/margins": 0.6376410722732544,
-      "rewards/rejected": -0.459087610244751,
-      "step": 320
-    },
-    {
-      "epoch": 0.34,
-      "learning_rate": 4.92545871559633e-07,
-      "logits/chosen": -2.6837377548217773,
-      "logits/rejected": -2.681138515472412,
-      "logps/chosen": -402.97576904296875,
-      "logps/rejected": -295.7830810546875,
-      "loss": 0.5687,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.2082200050354004,
-      "rewards/margins": 0.48193687200546265,
-      "rewards/rejected": -0.27371686697006226,
-      "step": 330
-    },
-    {
-      "epoch": 0.35,
-      "learning_rate": 4.906345565749235e-07,
-      "logits/chosen": -2.5517609119415283,
-      "logits/rejected": -2.561235189437866,
-      "logps/chosen": -300.0711364746094,
-      "logps/rejected": -265.93634033203125,
-      "loss": 0.5795,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.04220394045114517,
-      "rewards/margins": 0.34089353680610657,
-      "rewards/rejected": -0.2986895442008972,
-      "step": 340
-    },
-    {
-      "epoch": 0.36,
-      "learning_rate": 4.88723241590214e-07,
-      "logits/chosen": -2.6442813873291016,
-      "logits/rejected": -2.589465856552124,
-      "logps/chosen": -380.798583984375,
-      "logps/rejected": -283.53033447265625,
-      "loss": 0.5573,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.29559534788131714,
-      "rewards/margins": 0.628325343132019,
-      "rewards/rejected": -0.3327299952507019,
-      "step": 350
-    },
-    {
-      "epoch": 0.37,
-      "learning_rate": 4.868119266055046e-07,
-      "logits/chosen": -2.6972787380218506,
-      "logits/rejected": -2.6877760887145996,
-      "logps/chosen": -358.97381591796875,
-      "logps/rejected": -316.7508239746094,
-      "loss": 0.5399,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.135372132062912,
-      "rewards/margins": 0.6987503170967102,
-      "rewards/rejected": -0.5633782148361206,
-      "step": 360
-    },
-    {
-      "epoch": 0.38,
-      "learning_rate": 4.849006116207951e-07,
-      "logits/chosen": -2.7523036003112793,
-      "logits/rejected": -2.6924734115600586,
-      "logps/chosen": -357.83648681640625,
-      "logps/rejected": -301.96282958984375,
-      "loss": 0.5602,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.14637404680252075,
-      "rewards/margins": 0.7356597781181335,
-      "rewards/rejected": -0.5892857909202576,
-      "step": 370
-    },
-    {
-      "epoch": 0.39,
-      "learning_rate": 4.829892966360856e-07,
-      "logits/chosen": -2.696434736251831,
-      "logits/rejected": -2.6464056968688965,
-      "logps/chosen": -391.2596740722656,
-      "logps/rejected": -322.278564453125,
-      "loss": 0.5616,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.20244574546813965,
-      "rewards/margins": 0.6445008516311646,
-      "rewards/rejected": -0.4420550763607025,
-      "step": 380
-    },
-    {
-      "epoch": 0.4,
-      "learning_rate": 4.810779816513762e-07,
-      "logits/chosen": -2.6600687503814697,
-      "logits/rejected": -2.6298000812530518,
-      "logps/chosen": -298.4252014160156,
-      "logps/rejected": -267.12408447265625,
-      "loss": 0.5745,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.009316349402070045,
-      "rewards/margins": 0.48411574959754944,
-      "rewards/rejected": -0.47479939460754395,
-      "step": 390
-    },
-    {
-      "epoch": 0.41,
-      "learning_rate": 4.791666666666667e-07,
-      "logits/chosen": -2.6604857444763184,
-      "logits/rejected": -2.611046552658081,
-      "logps/chosen": -340.8629455566406,
-      "logps/rejected": -230.5844268798828,
-      "loss": 0.5633,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.2387767732143402,
-      "rewards/margins": 0.7385751008987427,
-      "rewards/rejected": -0.4997982978820801,
-      "step": 400
-    },
-    {
-      "epoch": 0.41,
-      "eval_logits/chosen": -2.7724692821502686,
-      "eval_logits/rejected": -2.709975004196167,
-      "eval_logps/chosen": -367.6446533203125,
-      "eval_logps/rejected": -290.0246276855469,
-      "eval_loss": 0.5475609302520752,
-      "eval_rewards/accuracies": 0.7103174328804016,
-      "eval_rewards/chosen": 0.22107475996017456,
-      "eval_rewards/margins": 0.6523182392120361,
-      "eval_rewards/rejected": -0.4312434792518616,
-      "eval_runtime": 93.5919,
-      "eval_samples_per_second": 21.369,
-      "eval_steps_per_second": 0.673,
-      "step": 400
-    },
-    {
-      "epoch": 0.42,
-      "learning_rate": 4.772553516819572e-07,
-      "logits/chosen": -2.6370530128479004,
-      "logits/rejected": -2.622492551803589,
-      "logps/chosen": -367.6494445800781,
-      "logps/rejected": -319.9653625488281,
-      "loss": 0.575,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.20758125185966492,
-      "rewards/margins": 0.6940993070602417,
-      "rewards/rejected": -0.4865179657936096,
-      "step": 410
-    },
-    {
-      "epoch": 0.43,
-      "learning_rate": 4.753440366972477e-07,
-      "logits/chosen": -2.714916706085205,
-      "logits/rejected": -2.6454455852508545,
-      "logps/chosen": -287.719970703125,
-      "logps/rejected": -275.41241455078125,
-      "loss": 0.5676,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.04725194722414017,
-      "rewards/margins": 0.4273280203342438,
-      "rewards/rejected": -0.38007602095603943,
-      "step": 420
-    },
-    {
-      "epoch": 0.44,
-      "learning_rate": 4.7343272171253825e-07,
-      "logits/chosen": -2.6437063217163086,
-      "logits/rejected": -2.648123264312744,
-      "logps/chosen": -305.0550537109375,
-      "logps/rejected": -266.63525390625,
-      "loss": 0.5725,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.03232135996222496,
-      "rewards/margins": 0.46253544092178345,
-      "rewards/rejected": -0.4302140772342682,
-      "step": 430
-    },
-    {
-      "epoch": 0.45,
-      "learning_rate": 4.715214067278288e-07,
-      "logits/chosen": -2.6272292137145996,
-      "logits/rejected": -2.6141581535339355,
-      "logps/chosen": -357.75750732421875,
-      "logps/rejected": -256.28765869140625,
-      "loss": 0.5434,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.0530555360019207,
-      "rewards/margins": 0.6745840311050415,
-      "rewards/rejected": -0.6215284466743469,
-      "step": 440
-    },
-    {
-      "epoch": 0.46,
-      "learning_rate": 4.696100917431192e-07,
-      "logits/chosen": -2.6640782356262207,
-      "logits/rejected": -2.661160707473755,
-      "logps/chosen": -395.8831481933594,
-      "logps/rejected": -325.2682800292969,
-      "loss": 0.5776,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.17429330945014954,
-      "rewards/margins": 0.49202069640159607,
-      "rewards/rejected": -0.3177274167537689,
-      "step": 450
-    },
-    {
-      "epoch": 0.47,
-      "learning_rate": 4.6769877675840974e-07,
-      "logits/chosen": -2.633216619491577,
-      "logits/rejected": -2.6277575492858887,
-      "logps/chosen": -319.4475402832031,
-      "logps/rejected": -281.19732666015625,
-      "loss": 0.529,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.060733817517757416,
-      "rewards/margins": 0.5816834568977356,
-      "rewards/rejected": -0.52094966173172,
-      "step": 460
-    },
-    {
-      "epoch": 0.49,
-      "learning_rate": 4.6578746177370027e-07,
-      "logits/chosen": -2.5858516693115234,
-      "logits/rejected": -2.562450647354126,
-      "logps/chosen": -253.75051879882812,
-      "logps/rejected": -221.74124145507812,
-      "loss": 0.5253,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 0.15773826837539673,
-      "rewards/margins": 0.6320953965187073,
-      "rewards/rejected": -0.47435712814331055,
-      "step": 470
-    },
-    {
-      "epoch": 0.5,
-      "learning_rate": 4.638761467889908e-07,
-      "logits/chosen": -2.6325182914733887,
-      "logits/rejected": -2.6154775619506836,
-      "logps/chosen": -358.68377685546875,
-      "logps/rejected": -252.25399780273438,
-      "loss": 0.5569,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.12426266819238663,
-      "rewards/margins": 0.7980905175209045,
-      "rewards/rejected": -0.6738277673721313,
-      "step": 480
-    },
-    {
-      "epoch": 0.51,
-      "learning_rate": 4.6196483180428133e-07,
-      "logits/chosen": -2.694267511367798,
-      "logits/rejected": -2.6052515506744385,
-      "logps/chosen": -343.802490234375,
-      "logps/rejected": -267.68292236328125,
-      "loss": 0.5419,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.17378923296928406,
-      "rewards/margins": 0.6964474320411682,
-      "rewards/rejected": -0.5226581692695618,
-      "step": 490
-    },
-    {
-      "epoch": 0.52,
-      "learning_rate": 4.600535168195718e-07,
-      "logits/chosen": -2.5756261348724365,
-      "logits/rejected": -2.5577731132507324,
-      "logps/chosen": -269.1250305175781,
-      "logps/rejected": -254.8256072998047,
-      "loss": 0.5224,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.12170092016458511,
-      "rewards/margins": 0.486471951007843,
-      "rewards/rejected": -0.3647710084915161,
-      "step": 500
-    },
-    {
-      "epoch": 0.52,
-      "eval_logits/chosen": -2.7543344497680664,
-      "eval_logits/rejected": -2.6918773651123047,
-      "eval_logps/chosen": -367.15386962890625,
-      "eval_logps/rejected": -290.25469970703125,
-      "eval_loss": 0.5388306975364685,
-      "eval_rewards/accuracies": 0.6984127163887024,
-      "eval_rewards/chosen": 0.2701512277126312,
-      "eval_rewards/margins": 0.7244044542312622,
-      "eval_rewards/rejected": -0.45425331592559814,
-      "eval_runtime": 93.436,
-      "eval_samples_per_second": 21.405,
-      "eval_steps_per_second": 0.674,
-      "step": 500
-    },
-    {
-      "epoch": 0.53,
-      "learning_rate": 4.5814220183486234e-07,
-      "logits/chosen": -2.5832715034484863,
-      "logits/rejected": -2.5189692974090576,
-      "logps/chosen": -355.7542419433594,
-      "logps/rejected": -309.67449951171875,
-      "loss": 0.5487,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.03541551157832146,
-      "rewards/margins": 0.4255645275115967,
-      "rewards/rejected": -0.39014899730682373,
-      "step": 510
-    },
-    {
-      "epoch": 0.54,
-      "learning_rate": 4.562308868501529e-07,
-      "logits/chosen": -2.6220099925994873,
-      "logits/rejected": -2.5353286266326904,
-      "logps/chosen": -334.10565185546875,
-      "logps/rejected": -302.77496337890625,
-      "loss": 0.5586,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.17356789112091064,
-      "rewards/margins": 0.6331679224967957,
-      "rewards/rejected": -0.459600031375885,
-      "step": 520
-    },
-    {
-      "epoch": 0.55,
-      "learning_rate": 4.543195718654434e-07,
-      "logits/chosen": -2.5556163787841797,
-      "logits/rejected": -2.600917100906372,
-      "logps/chosen": -282.23663330078125,
-      "logps/rejected": -241.3870086669922,
-      "loss": 0.5589,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.048130545765161514,
-      "rewards/margins": 0.40956515073776245,
-      "rewards/rejected": -0.36143457889556885,
-      "step": 530
-    },
-    {
-      "epoch": 0.56,
-      "learning_rate": 4.5240825688073394e-07,
-      "logits/chosen": -2.601301908493042,
-      "logits/rejected": -2.5350029468536377,
-      "logps/chosen": -348.8049011230469,
-      "logps/rejected": -257.8331298828125,
-      "loss": 0.555,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.32524555921554565,
-      "rewards/margins": 0.8066871762275696,
-      "rewards/rejected": -0.48144158720970154,
-      "step": 540
-    },
-    {
-      "epoch": 0.57,
-      "learning_rate": 4.504969418960244e-07,
-      "logits/chosen": -2.665917158126831,
-      "logits/rejected": -2.491896867752075,
-      "logps/chosen": -324.9315490722656,
-      "logps/rejected": -314.4938049316406,
-      "loss": 0.5501,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.21840611100196838,
-      "rewards/margins": 0.5038217306137085,
-      "rewards/rejected": -0.2854156792163849,
-      "step": 550
-    },
-    {
-      "epoch": 0.58,
-      "learning_rate": 4.4858562691131495e-07,
-      "logits/chosen": -2.666839599609375,
-      "logits/rejected": -2.6258294582366943,
-      "logps/chosen": -349.91668701171875,
-      "logps/rejected": -336.1511535644531,
-      "loss": 0.5326,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.2688800096511841,
-      "rewards/margins": 0.6696378588676453,
-      "rewards/rejected": -0.40075787901878357,
-      "step": 560
-    },
-    {
-      "epoch": 0.59,
-      "learning_rate": 4.466743119266055e-07,
-      "logits/chosen": -2.6923105716705322,
-      "logits/rejected": -2.6402506828308105,
-      "logps/chosen": -340.2569274902344,
-      "logps/rejected": -279.79827880859375,
-      "loss": 0.5463,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.08635896444320679,
-      "rewards/margins": 0.5042712688446045,
-      "rewards/rejected": -0.4179123342037201,
-      "step": 570
-    },
-    {
-      "epoch": 0.6,
-      "learning_rate": 4.44762996941896e-07,
-      "logits/chosen": -2.6503794193267822,
-      "logits/rejected": -2.6499364376068115,
-      "logps/chosen": -359.00439453125,
-      "logps/rejected": -300.85906982421875,
-      "loss": 0.5311,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.3588196337223053,
-      "rewards/margins": 0.8982511758804321,
-      "rewards/rejected": -0.5394314527511597,
-      "step": 580
-    },
-    {
-      "epoch": 0.61,
-      "learning_rate": 4.4285168195718655e-07,
-      "logits/chosen": -2.5828802585601807,
-      "logits/rejected": -2.551640033721924,
-      "logps/chosen": -295.88592529296875,
-      "logps/rejected": -282.87530517578125,
-      "loss": 0.5254,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.11026722192764282,
-      "rewards/margins": 0.5960223078727722,
-      "rewards/rejected": -0.4857551157474518,
-      "step": 590
-    },
-    {
-      "epoch": 0.62,
-      "learning_rate": 4.40940366972477e-07,
-      "logits/chosen": -2.7075865268707275,
-      "logits/rejected": -2.653106689453125,
-      "logps/chosen": -296.15606689453125,
-      "logps/rejected": -260.5872802734375,
-      "loss": 0.5689,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.1281842589378357,
-      "rewards/margins": 0.4070327877998352,
-      "rewards/rejected": -0.2788485288619995,
-      "step": 600
-    },
-    {
-      "epoch": 0.62,
-      "eval_logits/chosen": -2.7595744132995605,
-      "eval_logits/rejected": -2.69767689704895,
-      "eval_logps/chosen": -366.69464111328125,
-      "eval_logps/rejected": -290.02459716796875,
-      "eval_loss": 0.5326046347618103,
-      "eval_rewards/accuracies": 0.7301587462425232,
-      "eval_rewards/chosen": 0.31607678532600403,
-      "eval_rewards/margins": 0.7473164200782776,
-      "eval_rewards/rejected": -0.4312395751476288,
-      "eval_runtime": 93.591,
-      "eval_samples_per_second": 21.37,
-      "eval_steps_per_second": 0.673,
-      "step": 600
-    },
-    {
-      "epoch": 0.63,
-      "learning_rate": 4.3902905198776756e-07,
-      "logits/chosen": -2.6507742404937744,
-      "logits/rejected": -2.6280486583709717,
-      "logps/chosen": -353.1117248535156,
-      "logps/rejected": -269.77056884765625,
-      "loss": 0.5539,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.27281785011291504,
-      "rewards/margins": 0.7093775272369385,
-      "rewards/rejected": -0.43655967712402344,
-      "step": 610
-    },
-    {
-      "epoch": 0.64,
-      "learning_rate": 4.371177370030581e-07,
-      "logits/chosen": -2.660865306854248,
-      "logits/rejected": -2.587761163711548,
-      "logps/chosen": -347.13623046875,
-      "logps/rejected": -274.3702697753906,
-      "loss": 0.5253,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.25370901823043823,
-      "rewards/margins": 0.7508881688117981,
-      "rewards/rejected": -0.4971791207790375,
-      "step": 620
-    },
-    {
-      "epoch": 0.65,
-      "learning_rate": 4.352064220183486e-07,
-      "logits/chosen": -2.582853078842163,
-      "logits/rejected": -2.5666868686676025,
-      "logps/chosen": -275.5951232910156,
-      "logps/rejected": -243.16189575195312,
-      "loss": 0.5386,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.18081825971603394,
-      "rewards/margins": 0.5823779106140137,
-      "rewards/rejected": -0.4015596807003021,
-      "step": 630
-    },
-    {
-      "epoch": 0.66,
-      "learning_rate": 4.3329510703363915e-07,
-      "logits/chosen": -2.689371109008789,
-      "logits/rejected": -2.618781566619873,
-      "logps/chosen": -328.7333984375,
-      "logps/rejected": -252.730712890625,
-      "loss": 0.5306,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.21093907952308655,
-      "rewards/margins": 0.7845473885536194,
-      "rewards/rejected": -0.5736082792282104,
-      "step": 640
-    },
-    {
-      "epoch": 0.67,
-      "learning_rate": 4.313837920489297e-07,
-      "logits/chosen": -2.5996086597442627,
-      "logits/rejected": -2.6289618015289307,
-      "logps/chosen": -293.96502685546875,
-      "logps/rejected": -266.9768981933594,
-      "loss": 0.5293,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.2461429089307785,
-      "rewards/margins": 0.7484090924263,
-      "rewards/rejected": -0.5022662878036499,
-      "step": 650
-    },
-    {
-      "epoch": 0.68,
-      "learning_rate": 4.2947247706422016e-07,
-      "logits/chosen": -2.575277328491211,
-      "logits/rejected": -2.627274751663208,
-      "logps/chosen": -324.3821105957031,
-      "logps/rejected": -280.3650817871094,
-      "loss": 0.5277,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.10573621094226837,
-      "rewards/margins": 0.7033983469009399,
-      "rewards/rejected": -0.5976621508598328,
-      "step": 660
-    },
-    {
-      "epoch": 0.69,
-      "learning_rate": 4.275611620795107e-07,
-      "logits/chosen": -2.6651196479797363,
-      "logits/rejected": -2.5892019271850586,
-      "logps/chosen": -396.19073486328125,
-      "logps/rejected": -298.2142333984375,
-      "loss": 0.5287,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.17894011735916138,
-      "rewards/margins": 0.7362491488456726,
-      "rewards/rejected": -0.5573090314865112,
-      "step": 670
-    },
-    {
-      "epoch": 0.7,
-      "learning_rate": 4.2564984709480123e-07,
-      "logits/chosen": -2.682351589202881,
-      "logits/rejected": -2.6517555713653564,
-      "logps/chosen": -359.2548828125,
-      "logps/rejected": -272.7398986816406,
-      "loss": 0.584,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.16825886070728302,
-      "rewards/margins": 0.7031353712081909,
-      "rewards/rejected": -0.5348765254020691,
-      "step": 680
-    },
-    {
-      "epoch": 0.71,
-      "learning_rate": 4.2373853211009176e-07,
-      "logits/chosen": -2.652602195739746,
-      "logits/rejected": -2.6651628017425537,
-      "logps/chosen": -333.54949951171875,
-      "logps/rejected": -288.21405029296875,
-      "loss": 0.5643,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.35042139887809753,
-      "rewards/margins": 0.7546127438545227,
-      "rewards/rejected": -0.40419134497642517,
-      "step": 690
-    },
-    {
-      "epoch": 0.72,
-      "learning_rate": 4.2182721712538224e-07,
-      "logits/chosen": -2.690903902053833,
-      "logits/rejected": -2.5803565979003906,
-      "logps/chosen": -371.2886657714844,
-      "logps/rejected": -264.31353759765625,
-      "loss": 0.5556,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.4146875739097595,
-      "rewards/margins": 0.7678929567337036,
-      "rewards/rejected": -0.35320529341697693,
-      "step": 700
-    },
-    {
-      "epoch": 0.72,
-      "eval_logits/chosen": -2.7563326358795166,
-      "eval_logits/rejected": -2.6959750652313232,
-      "eval_logps/chosen": -366.72216796875,
-      "eval_logps/rejected": -290.1435852050781,
-      "eval_loss": 0.5295916795730591,
-      "eval_rewards/accuracies": 0.7142857313156128,
-      "eval_rewards/chosen": 0.31332191824913025,
-      "eval_rewards/margins": 0.7564623355865479,
-      "eval_rewards/rejected": -0.4431404173374176,
-      "eval_runtime": 93.2482,
-      "eval_samples_per_second": 21.448,
-      "eval_steps_per_second": 0.676,
-      "step": 700
-    },
-    {
-      "epoch": 0.73,
-      "learning_rate": 4.199159021406727e-07,
-      "logits/chosen": -2.6533806324005127,
-      "logits/rejected": -2.539382219314575,
-      "logps/chosen": -305.8080139160156,
-      "logps/rejected": -260.7340393066406,
-      "loss": 0.5391,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.2109827995300293,
-      "rewards/margins": 0.7150342464447021,
-      "rewards/rejected": -0.5040515065193176,
-      "step": 710
-    },
-    {
-      "epoch": 0.74,
-      "learning_rate": 4.1800458715596325e-07,
-      "logits/chosen": -2.651778221130371,
-      "logits/rejected": -2.5990653038024902,
-      "logps/chosen": -392.23675537109375,
-      "logps/rejected": -305.5450439453125,
-      "loss": 0.5729,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.2182011604309082,
-      "rewards/margins": 0.5736824870109558,
-      "rewards/rejected": -0.35548135638237,
-      "step": 720
-    },
-    {
-      "epoch": 0.75,
-      "learning_rate": 4.160932721712538e-07,
-      "logits/chosen": -2.6589245796203613,
-      "logits/rejected": -2.6071112155914307,
-      "logps/chosen": -344.4822692871094,
-      "logps/rejected": -308.5715026855469,
-      "loss": 0.5362,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.2045893371105194,
-      "rewards/margins": 0.5438891649246216,
-      "rewards/rejected": -0.339299738407135,
-      "step": 730
-    },
-    {
-      "epoch": 0.76,
-      "learning_rate": 4.141819571865443e-07,
-      "logits/chosen": -2.6027140617370605,
-      "logits/rejected": -2.5531044006347656,
-      "logps/chosen": -308.57275390625,
-      "logps/rejected": -227.69961547851562,
-      "loss": 0.5297,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.24511054158210754,
-      "rewards/margins": 0.7321642637252808,
-      "rewards/rejected": -0.4870537221431732,
-      "step": 740
-    },
-    {
-      "epoch": 0.77,
-      "learning_rate": 4.1227064220183485e-07,
-      "logits/chosen": -2.593599796295166,
-      "logits/rejected": -2.551053524017334,
-      "logps/chosen": -329.5063171386719,
-      "logps/rejected": -271.2137756347656,
-      "loss": 0.5408,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.058064889162778854,
-      "rewards/margins": 0.7626748085021973,
-      "rewards/rejected": -0.7046098113059998,
-      "step": 750
-    },
-    {
-      "epoch": 0.78,
-      "learning_rate": 4.103593272171253e-07,
-      "logits/chosen": -2.5975751876831055,
-      "logits/rejected": -2.611335515975952,
-      "logps/chosen": -338.27105712890625,
-      "logps/rejected": -318.44110107421875,
-      "loss": 0.5401,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.15539416670799255,
-      "rewards/margins": 0.6962918639183044,
-      "rewards/rejected": -0.5408977270126343,
-      "step": 760
-    },
-    {
-      "epoch": 0.79,
-      "learning_rate": 4.0844801223241586e-07,
-      "logits/chosen": -2.5972952842712402,
-      "logits/rejected": -2.5851306915283203,
-      "logps/chosen": -358.0859375,
-      "logps/rejected": -290.6552734375,
-      "loss": 0.5151,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.3178876042366028,
-      "rewards/margins": 0.6411694288253784,
-      "rewards/rejected": -0.3232818841934204,
-      "step": 770
-    },
-    {
-      "epoch": 0.8,
-      "learning_rate": 4.065366972477064e-07,
-      "logits/chosen": -2.6409897804260254,
-      "logits/rejected": -2.620407819747925,
-      "logps/chosen": -362.7335510253906,
-      "logps/rejected": -288.424072265625,
-      "loss": 0.5424,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.21879442036151886,
-      "rewards/margins": 0.7934145927429199,
-      "rewards/rejected": -0.574620246887207,
-      "step": 780
-    },
-    {
-      "epoch": 0.82,
-      "learning_rate": 4.046253822629969e-07,
-      "logits/chosen": -2.6204047203063965,
-      "logits/rejected": -2.575136661529541,
-      "logps/chosen": -335.6562194824219,
-      "logps/rejected": -271.09759521484375,
-      "loss": 0.5051,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.2648167610168457,
-      "rewards/margins": 0.7104849815368652,
-      "rewards/rejected": -0.44566813111305237,
-      "step": 790
-    },
-    {
-      "epoch": 0.83,
-      "learning_rate": 4.0271406727828745e-07,
-      "logits/chosen": -2.650829792022705,
-      "logits/rejected": -2.639927864074707,
-      "logps/chosen": -326.4569091796875,
-      "logps/rejected": -280.28082275390625,
-      "loss": 0.5368,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.19319041073322296,
-      "rewards/margins": 0.7100410461425781,
-      "rewards/rejected": -0.5168507099151611,
-      "step": 800
-    },
-    {
-      "epoch": 0.83,
-      "eval_logits/chosen": -2.7454872131347656,
-      "eval_logits/rejected": -2.6863038539886475,
-      "eval_logps/chosen": -366.76788330078125,
-      "eval_logps/rejected": -290.7203063964844,
-      "eval_loss": 0.5235481858253479,
-      "eval_rewards/accuracies": 0.7182539701461792,
-      "eval_rewards/chosen": 0.3087466061115265,
-      "eval_rewards/margins": 0.8095585107803345,
-      "eval_rewards/rejected": -0.5008119344711304,
-      "eval_runtime": 93.6963,
-      "eval_samples_per_second": 21.346,
-      "eval_steps_per_second": 0.672,
-      "step": 800
-    },
-    {
-      "epoch": 0.84,
-      "learning_rate": 4.00802752293578e-07,
-      "logits/chosen": -2.605900526046753,
-      "logits/rejected": -2.555630922317505,
-      "logps/chosen": -345.3916931152344,
-      "logps/rejected": -285.39794921875,
-      "loss": 0.5059,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.22336134314537048,
-      "rewards/margins": 0.6163698434829712,
-      "rewards/rejected": -0.3930084705352783,
-      "step": 810
-    },
-    {
-      "epoch": 0.85,
-      "learning_rate": 3.9889143730886847e-07,
-      "logits/chosen": -2.7067675590515137,
-      "logits/rejected": -2.6543965339660645,
-      "logps/chosen": -391.22271728515625,
-      "logps/rejected": -301.2769775390625,
-      "loss": 0.5472,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.2929418683052063,
-      "rewards/margins": 0.7365559339523315,
-      "rewards/rejected": -0.44361400604248047,
-      "step": 820
-    },
-    {
-      "epoch": 0.86,
-      "learning_rate": 3.96980122324159e-07,
-      "logits/chosen": -2.6316628456115723,
-      "logits/rejected": -2.583803653717041,
-      "logps/chosen": -328.14593505859375,
-      "logps/rejected": -247.4641876220703,
-      "loss": 0.5115,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.29103153944015503,
-      "rewards/margins": 0.9226154088973999,
-      "rewards/rejected": -0.6315839886665344,
-      "step": 830
-    },
-    {
-      "epoch": 0.87,
-      "learning_rate": 3.9506880733944953e-07,
-      "logits/chosen": -2.62970232963562,
-      "logits/rejected": -2.5595858097076416,
-      "logps/chosen": -312.72320556640625,
-      "logps/rejected": -264.06927490234375,
-      "loss": 0.569,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.2118232697248459,
-      "rewards/margins": 0.8576194047927856,
-      "rewards/rejected": -0.6457961797714233,
-      "step": 840
-    },
-    {
-      "epoch": 0.88,
-      "learning_rate": 3.9315749235474006e-07,
-      "logits/chosen": -2.5976932048797607,
-      "logits/rejected": -2.6474385261535645,
-      "logps/chosen": -352.48345947265625,
-      "logps/rejected": -319.6570739746094,
-      "loss": 0.5311,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.2210746556520462,
-      "rewards/margins": 0.6267276406288147,
-      "rewards/rejected": -0.4056529402732849,
-      "step": 850
-    },
-    {
-      "epoch": 0.89,
-      "learning_rate": 3.912461773700306e-07,
-      "logits/chosen": -2.7138149738311768,
-      "logits/rejected": -2.6776392459869385,
-      "logps/chosen": -351.1930236816406,
-      "logps/rejected": -328.28466796875,
-      "loss": 0.5424,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.16447851061820984,
-      "rewards/margins": 0.7863180637359619,
-      "rewards/rejected": -0.6218395233154297,
-      "step": 860
-    },
-    {
-      "epoch": 0.9,
-      "learning_rate": 3.8933486238532107e-07,
-      "logits/chosen": -2.493483304977417,
-      "logits/rejected": -2.503645420074463,
-      "logps/chosen": -380.6748046875,
-      "logps/rejected": -284.23272705078125,
-      "loss": 0.5245,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.2832086980342865,
-      "rewards/margins": 0.8768035173416138,
-      "rewards/rejected": -0.5935948491096497,
-      "step": 870
-    },
-    {
-      "epoch": 0.91,
-      "learning_rate": 3.874235474006116e-07,
-      "logits/chosen": -2.607677936553955,
-      "logits/rejected": -2.5798025131225586,
-      "logps/chosen": -332.20635986328125,
-      "logps/rejected": -271.32281494140625,
-      "loss": 0.5417,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.297354131937027,
-      "rewards/margins": 0.7697429656982422,
-      "rewards/rejected": -0.4723888337612152,
-      "step": 880
-    },
-    {
-      "epoch": 0.92,
-      "learning_rate": 3.8551223241590214e-07,
-      "logits/chosen": -2.6368677616119385,
-      "logits/rejected": -2.571981906890869,
-      "logps/chosen": -321.2016906738281,
-      "logps/rejected": -274.25274658203125,
-      "loss": 0.5424,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.2311544120311737,
-      "rewards/margins": 0.679220974445343,
-      "rewards/rejected": -0.4480666220188141,
-      "step": 890
-    },
-    {
-      "epoch": 0.93,
-      "learning_rate": 3.8360091743119267e-07,
-      "logits/chosen": -2.6375088691711426,
-      "logits/rejected": -2.5301380157470703,
-      "logps/chosen": -313.16973876953125,
-      "logps/rejected": -271.7088623046875,
-      "loss": 0.5324,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.27663588523864746,
-      "rewards/margins": 0.7922950983047485,
-      "rewards/rejected": -0.5156592130661011,
-      "step": 900
-    },
-    {
-      "epoch": 0.93,
-      "eval_logits/chosen": -2.753173351287842,
-      "eval_logits/rejected": -2.694375991821289,
-      "eval_logps/chosen": -366.52520751953125,
-      "eval_logps/rejected": -290.4763488769531,
-      "eval_loss": 0.5231475830078125,
-      "eval_rewards/accuracies": 0.738095223903656,
-      "eval_rewards/chosen": 0.3330167233943939,
-      "eval_rewards/margins": 0.809434175491333,
-      "eval_rewards/rejected": -0.4764174818992615,
-      "eval_runtime": 93.6005,
-      "eval_samples_per_second": 21.367,
-      "eval_steps_per_second": 0.673,
-      "step": 900
-    },
-    {
-      "epoch": 0.94,
-      "learning_rate": 3.816896024464832e-07,
-      "logits/chosen": -2.628507614135742,
-      "logits/rejected": -2.6525845527648926,
-      "logps/chosen": -281.3057861328125,
-      "logps/rejected": -249.5850067138672,
-      "loss": 0.558,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.2345593422651291,
-      "rewards/margins": 0.6988226771354675,
-      "rewards/rejected": -0.464263379573822,
-      "step": 910
-    },
-    {
-      "epoch": 0.95,
-      "learning_rate": 3.797782874617737e-07,
-      "logits/chosen": -2.667789936065674,
-      "logits/rejected": -2.6839444637298584,
-      "logps/chosen": -340.7831115722656,
-      "logps/rejected": -275.1526794433594,
-      "loss": 0.5423,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.1838233470916748,
-      "rewards/margins": 0.4514841139316559,
-      "rewards/rejected": -0.2676607668399811,
-      "step": 920
-    },
-    {
-      "epoch": 0.96,
-      "learning_rate": 3.778669724770642e-07,
-      "logits/chosen": -2.5922303199768066,
-      "logits/rejected": -2.578035831451416,
-      "logps/chosen": -319.7664489746094,
-      "logps/rejected": -242.1009063720703,
-      "loss": 0.5417,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.1281779408454895,
-      "rewards/margins": 0.6992352604866028,
-      "rewards/rejected": -0.5710573792457581,
-      "step": 930
-    },
-    {
-      "epoch": 0.97,
-      "learning_rate": 3.7595565749235474e-07,
-      "logits/chosen": -2.7459988594055176,
-      "logits/rejected": -2.6785151958465576,
-      "logps/chosen": -357.2470703125,
-      "logps/rejected": -257.43853759765625,
-      "loss": 0.5332,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.44489067792892456,
-      "rewards/margins": 0.813278317451477,
-      "rewards/rejected": -0.3683876097202301,
-      "step": 940
-    },
-    {
-      "epoch": 0.98,
-      "learning_rate": 3.740443425076452e-07,
-      "logits/chosen": -2.6817257404327393,
-      "logits/rejected": -2.6576712131500244,
-      "logps/chosen": -350.62200927734375,
-      "logps/rejected": -299.66448974609375,
-      "loss": 0.5362,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.2472398281097412,
-      "rewards/margins": 0.6820510625839233,
-      "rewards/rejected": -0.4348112642765045,
-      "step": 950
-    },
-    {
-      "epoch": 0.99,
-      "learning_rate": 3.7213302752293575e-07,
-      "logits/chosen": -2.646120309829712,
-      "logits/rejected": -2.607883930206299,
-      "logps/chosen": -338.7344665527344,
-      "logps/rejected": -275.9519958496094,
-      "loss": 0.5097,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.32568275928497314,
-      "rewards/margins": 0.5753262639045715,
-      "rewards/rejected": -0.24964356422424316,
-      "step": 960
-    },
-    {
-      "epoch": 1.0,
-      "learning_rate": 3.702217125382263e-07,
-      "logits/chosen": -2.649812698364258,
-      "logits/rejected": -2.5741024017333984,
-      "logps/chosen": -369.88861083984375,
-      "logps/rejected": -302.81536865234375,
-      "loss": 0.4998,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.188663512468338,
-      "rewards/margins": 0.8093651533126831,
-      "rewards/rejected": -0.6207016110420227,
-      "step": 970
-    },
-    {
-      "epoch": 1.01,
-      "learning_rate": 3.6831039755351677e-07,
-      "logits/chosen": -2.6051228046417236,
-      "logits/rejected": -2.6672070026397705,
-      "logps/chosen": -333.53570556640625,
-      "logps/rejected": -285.21490478515625,
-      "loss": 0.4345,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.42203760147094727,
-      "rewards/margins": 1.160388708114624,
-      "rewards/rejected": -0.7383512258529663,
-      "step": 980
-    },
-    {
-      "epoch": 1.02,
-      "learning_rate": 3.663990825688073e-07,
-      "logits/chosen": -2.6160953044891357,
-      "logits/rejected": -2.553997755050659,
-      "logps/chosen": -319.94171142578125,
-      "logps/rejected": -277.0104064941406,
-      "loss": 0.4979,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.2790919244289398,
-      "rewards/margins": 0.8934859037399292,
-      "rewards/rejected": -0.6143940687179565,
-      "step": 990
-    },
-    {
-      "epoch": 1.03,
-      "learning_rate": 3.6448776758409783e-07,
-      "logits/chosen": -2.7189905643463135,
-      "logits/rejected": -2.594120979309082,
-      "logps/chosen": -345.72369384765625,
-      "logps/rejected": -298.8501281738281,
-      "loss": 0.4667,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 0.45513787865638733,
-      "rewards/margins": 0.9279818534851074,
-      "rewards/rejected": -0.4728439450263977,
-      "step": 1000
-    },
-    {
-      "epoch": 1.03,
-      "eval_logits/chosen": -2.7465925216674805,
-      "eval_logits/rejected": -2.688951253890991,
-      "eval_logps/chosen": -366.4130554199219,
-      "eval_logps/rejected": -290.52691650390625,
-      "eval_loss": 0.521090567111969,
-      "eval_rewards/accuracies": 0.7301587462425232,
-      "eval_rewards/chosen": 0.3442315459251404,
-      "eval_rewards/margins": 0.8257037997245789,
-      "eval_rewards/rejected": -0.48147228360176086,
-      "eval_runtime": 93.2289,
-      "eval_samples_per_second": 21.453,
-      "eval_steps_per_second": 0.676,
-      "step": 1000
-    },
-    {
-      "epoch": 1.04,
-      "learning_rate": 3.6257645259938836e-07,
-      "logits/chosen": -2.6213080883026123,
-      "logits/rejected": -2.6167352199554443,
-      "logps/chosen": -311.80169677734375,
-      "logps/rejected": -280.1716613769531,
-      "loss": 0.4443,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.372317373752594,
-      "rewards/margins": 0.9983639717102051,
-      "rewards/rejected": -0.6260465979576111,
-      "step": 1010
-    },
-    {
-      "epoch": 1.05,
-      "learning_rate": 3.606651376146789e-07,
-      "logits/chosen": -2.497844696044922,
-      "logits/rejected": -2.5120177268981934,
-      "logps/chosen": -311.4458312988281,
-      "logps/rejected": -230.55569458007812,
-      "loss": 0.4408,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.3275541663169861,
-      "rewards/margins": 0.951290488243103,
-      "rewards/rejected": -0.6237363815307617,
-      "step": 1020
-    },
-    {
-      "epoch": 1.06,
-      "learning_rate": 3.5875382262996937e-07,
-      "logits/chosen": -2.5676560401916504,
-      "logits/rejected": -2.576164722442627,
-      "logps/chosen": -336.22955322265625,
-      "logps/rejected": -327.7220153808594,
-      "loss": 0.4314,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.4555473327636719,
-      "rewards/margins": 0.9157599210739136,
-      "rewards/rejected": -0.4602126181125641,
-      "step": 1030
-    },
-    {
-      "epoch": 1.07,
-      "learning_rate": 3.568425076452599e-07,
-      "logits/chosen": -2.683770179748535,
-      "logits/rejected": -2.5734055042266846,
-      "logps/chosen": -348.1385192871094,
-      "logps/rejected": -290.2071838378906,
-      "loss": 0.4376,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.2006378471851349,
-      "rewards/margins": 0.8747081756591797,
-      "rewards/rejected": -0.6740702986717224,
-      "step": 1040
-    },
-    {
-      "epoch": 1.08,
-      "learning_rate": 3.5493119266055044e-07,
-      "logits/chosen": -2.529114246368408,
-      "logits/rejected": -2.5104918479919434,
-      "logps/chosen": -326.69842529296875,
-      "logps/rejected": -234.59725952148438,
-      "loss": 0.4363,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.2911790907382965,
-      "rewards/margins": 1.0217406749725342,
-      "rewards/rejected": -0.7305616140365601,
-      "step": 1050
-    },
-    {
-      "epoch": 1.09,
-      "learning_rate": 3.5301987767584097e-07,
-      "logits/chosen": -2.5719220638275146,
-      "logits/rejected": -2.503883123397827,
-      "logps/chosen": -332.20758056640625,
-      "logps/rejected": -299.2622375488281,
-      "loss": 0.4347,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.3985035717487335,
-      "rewards/margins": 1.1057422161102295,
-      "rewards/rejected": -0.7072386145591736,
-      "step": 1060
-    },
-    {
-      "epoch": 1.1,
-      "learning_rate": 3.511085626911315e-07,
-      "logits/chosen": -2.6049246788024902,
-      "logits/rejected": -2.4959778785705566,
-      "logps/chosen": -388.32818603515625,
-      "logps/rejected": -289.9732971191406,
-      "loss": 0.439,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 0.37873345613479614,
-      "rewards/margins": 1.0958116054534912,
-      "rewards/rejected": -0.7170782089233398,
-      "step": 1070
-    },
-    {
-      "epoch": 1.11,
-      "learning_rate": 3.49197247706422e-07,
-      "logits/chosen": -2.5549204349517822,
-      "logits/rejected": -2.5598814487457275,
-      "logps/chosen": -276.199951171875,
-      "logps/rejected": -245.2752685546875,
-      "loss": 0.4359,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.11906807124614716,
-      "rewards/margins": 0.9462801218032837,
-      "rewards/rejected": -0.8272121548652649,
-      "step": 1080
-    },
-    {
-      "epoch": 1.12,
-      "learning_rate": 3.472859327217125e-07,
-      "logits/chosen": -2.477860450744629,
-      "logits/rejected": -2.413344144821167,
-      "logps/chosen": -355.36309814453125,
-      "logps/rejected": -283.76123046875,
-      "loss": 0.4421,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 0.2770375609397888,
-      "rewards/margins": 1.1055151224136353,
-      "rewards/rejected": -0.8284775614738464,
-      "step": 1090
-    },
-    {
-      "epoch": 1.14,
-      "learning_rate": 3.4537461773700304e-07,
-      "logits/chosen": -2.53197979927063,
-      "logits/rejected": -2.5624704360961914,
-      "logps/chosen": -252.6802978515625,
-      "logps/rejected": -211.03775024414062,
-      "loss": 0.4516,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 0.12486497312784195,
-      "rewards/margins": 1.0134916305541992,
-      "rewards/rejected": -0.8886265754699707,
-      "step": 1100
-    },
-    {
-      "epoch": 1.14,
-      "eval_logits/chosen": -2.73252272605896,
-      "eval_logits/rejected": -2.6769962310791016,
-      "eval_logps/chosen": -367.01220703125,
-      "eval_logps/rejected": -291.74310302734375,
-      "eval_loss": 0.5197181105613708,
-      "eval_rewards/accuracies": 0.738095223903656,
-      "eval_rewards/chosen": 0.28431636095046997,
-      "eval_rewards/margins": 0.8874091506004333,
-      "eval_rewards/rejected": -0.6030928492546082,
-      "eval_runtime": 93.7509,
-      "eval_samples_per_second": 21.333,
-      "eval_steps_per_second": 0.672,
-      "step": 1100
-    },
-    {
-      "epoch": 1.15,
-      "learning_rate": 3.434633027522936e-07,
-      "logits/chosen": -2.629892587661743,
-      "logits/rejected": -2.5663561820983887,
-      "logps/chosen": -359.1500549316406,
-      "logps/rejected": -279.5158386230469,
-      "loss": 0.4234,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.4036155641078949,
-      "rewards/margins": 0.9569507837295532,
-      "rewards/rejected": -0.5533351898193359,
-      "step": 1110
-    },
-    {
-      "epoch": 1.16,
-      "learning_rate": 3.415519877675841e-07,
-      "logits/chosen": -2.5857465267181396,
-      "logits/rejected": -2.565286636352539,
-      "logps/chosen": -307.4760437011719,
-      "logps/rejected": -296.2510070800781,
-      "loss": 0.4412,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.31315886974334717,
-      "rewards/margins": 0.915540874004364,
-      "rewards/rejected": -0.6023820638656616,
-      "step": 1120
-    },
-    {
-      "epoch": 1.17,
-      "learning_rate": 3.3964067278287464e-07,
-      "logits/chosen": -2.638646364212036,
-      "logits/rejected": -2.590546131134033,
-      "logps/chosen": -354.6069641113281,
-      "logps/rejected": -261.17510986328125,
-      "loss": 0.4283,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 0.3109307289123535,
-      "rewards/margins": 1.081446647644043,
-      "rewards/rejected": -0.7705159187316895,
-      "step": 1130
-    },
-    {
-      "epoch": 1.18,
-      "learning_rate": 3.377293577981651e-07,
-      "logits/chosen": -2.5345458984375,
-      "logits/rejected": -2.5311672687530518,
-      "logps/chosen": -288.7183532714844,
-      "logps/rejected": -270.4634704589844,
-      "loss": 0.4028,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.10796526819467545,
-      "rewards/margins": 1.147101640701294,
-      "rewards/rejected": -1.0391361713409424,
-      "step": 1140
-    },
-    {
-      "epoch": 1.19,
-      "learning_rate": 3.3581804281345565e-07,
-      "logits/chosen": -2.6268527507781982,
-      "logits/rejected": -2.539853572845459,
-      "logps/chosen": -364.0658874511719,
-      "logps/rejected": -269.86798095703125,
-      "loss": 0.4622,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.19580404460430145,
-      "rewards/margins": 0.9232183694839478,
-      "rewards/rejected": -0.7274142503738403,
-      "step": 1150
-    },
-    {
-      "epoch": 1.2,
-      "learning_rate": 3.339067278287462e-07,
-      "logits/chosen": -2.4458861351013184,
-      "logits/rejected": -2.478278875350952,
-      "logps/chosen": -316.80975341796875,
-      "logps/rejected": -268.9418640136719,
-      "loss": 0.4354,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.21194472908973694,
-      "rewards/margins": 0.9436850547790527,
-      "rewards/rejected": -0.731740415096283,
-      "step": 1160
-    },
-    {
-      "epoch": 1.21,
-      "learning_rate": 3.319954128440367e-07,
-      "logits/chosen": -2.6692590713500977,
-      "logits/rejected": -2.565462827682495,
-      "logps/chosen": -363.7535400390625,
-      "logps/rejected": -267.73858642578125,
-      "loss": 0.4547,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.2941077649593353,
-      "rewards/margins": 1.0993130207061768,
-      "rewards/rejected": -0.8052051663398743,
-      "step": 1170
-    },
-    {
-      "epoch": 1.22,
-      "learning_rate": 3.3008409785932725e-07,
-      "logits/chosen": -2.55599308013916,
-      "logits/rejected": -2.5489039421081543,
-      "logps/chosen": -317.7440490722656,
-      "logps/rejected": -261.28021240234375,
-      "loss": 0.4364,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 0.14147642254829407,
-      "rewards/margins": 1.0507941246032715,
-      "rewards/rejected": -0.909317672252655,
-      "step": 1180
-    },
-    {
-      "epoch": 1.23,
-      "learning_rate": 3.2817278287461773e-07,
-      "logits/chosen": -2.5962812900543213,
-      "logits/rejected": -2.5714190006256104,
-      "logps/chosen": -291.72479248046875,
-      "logps/rejected": -285.7821350097656,
-      "loss": 0.421,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.2120455503463745,
-      "rewards/margins": 1.1750547885894775,
-      "rewards/rejected": -0.963009238243103,
-      "step": 1190
-    },
-    {
-      "epoch": 1.24,
-      "learning_rate": 3.262614678899082e-07,
-      "logits/chosen": -2.585196018218994,
-      "logits/rejected": -2.582301378250122,
-      "logps/chosen": -314.81842041015625,
-      "logps/rejected": -294.3887023925781,
-      "loss": 0.4176,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.03839557617902756,
-      "rewards/margins": 0.9020207524299622,
-      "rewards/rejected": -0.863625168800354,
-      "step": 1200
-    },
-    {
-      "epoch": 1.24,
-      "eval_logits/chosen": -2.7277021408081055,
-      "eval_logits/rejected": -2.672938108444214,
-      "eval_logps/chosen": -367.73968505859375,
-      "eval_logps/rejected": -292.8726806640625,
-      "eval_loss": 0.5184498429298401,
-      "eval_rewards/accuracies": 0.7460317611694336,
-      "eval_rewards/chosen": 0.21156975626945496,
-      "eval_rewards/margins": 0.9276205897331238,
-      "eval_rewards/rejected": -0.716050922870636,
-      "eval_runtime": 93.6697,
-      "eval_samples_per_second": 21.352,
-      "eval_steps_per_second": 0.673,
-      "step": 1200
-    },
-    {
-      "epoch": 1.25,
-      "learning_rate": 3.2435015290519874e-07,
-      "logits/chosen": -2.5604007244110107,
-      "logits/rejected": -2.522665023803711,
-      "logps/chosen": -278.86138916015625,
-      "logps/rejected": -238.6088104248047,
-      "loss": 0.4139,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.17497147619724274,
-      "rewards/margins": 1.1593296527862549,
-      "rewards/rejected": -0.9843581318855286,
-      "step": 1210
-    },
-    {
-      "epoch": 1.26,
-      "learning_rate": 3.2243883792048927e-07,
-      "logits/chosen": -2.6064276695251465,
-      "logits/rejected": -2.6227469444274902,
-      "logps/chosen": -359.59637451171875,
-      "logps/rejected": -339.55731201171875,
-      "loss": 0.4567,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.2552815079689026,
-      "rewards/margins": 1.0309327840805054,
-      "rewards/rejected": -0.7756513357162476,
-      "step": 1220
-    },
-    {
-      "epoch": 1.27,
-      "learning_rate": 3.205275229357798e-07,
-      "logits/chosen": -2.6077446937561035,
-      "logits/rejected": -2.5000662803649902,
-      "logps/chosen": -361.3202819824219,
-      "logps/rejected": -317.98504638671875,
-      "loss": 0.4455,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.11873336881399155,
-      "rewards/margins": 0.9340359568595886,
-      "rewards/rejected": -0.8153026700019836,
-      "step": 1230
-    },
-    {
-      "epoch": 1.28,
-      "learning_rate": 3.186162079510703e-07,
-      "logits/chosen": -2.6293277740478516,
-      "logits/rejected": -2.5831921100616455,
-      "logps/chosen": -347.88690185546875,
-      "logps/rejected": -309.56500244140625,
-      "loss": 0.4596,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.2751849293708801,
-      "rewards/margins": 1.015895962715149,
-      "rewards/rejected": -0.7407109141349792,
-      "step": 1240
-    },
-    {
-      "epoch": 1.29,
-      "learning_rate": 3.167048929663608e-07,
-      "logits/chosen": -2.574522018432617,
-      "logits/rejected": -2.5256123542785645,
-      "logps/chosen": -294.55194091796875,
-      "logps/rejected": -239.9694061279297,
-      "loss": 0.4325,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.2018485963344574,
-      "rewards/margins": 1.1163266897201538,
-      "rewards/rejected": -0.9144781827926636,
-      "step": 1250
-    },
-    {
-      "epoch": 1.3,
-      "learning_rate": 3.1479357798165134e-07,
-      "logits/chosen": -2.658360242843628,
-      "logits/rejected": -2.649301290512085,
-      "logps/chosen": -352.422607421875,
-      "logps/rejected": -288.5453186035156,
-      "loss": 0.4301,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.26997870206832886,
-      "rewards/margins": 1.0690968036651611,
-      "rewards/rejected": -0.7991181015968323,
-      "step": 1260
-    },
-    {
-      "epoch": 1.31,
-      "learning_rate": 3.128822629969419e-07,
-      "logits/chosen": -2.658994197845459,
-      "logits/rejected": -2.6330344676971436,
-      "logps/chosen": -349.5880432128906,
-      "logps/rejected": -295.90374755859375,
-      "loss": 0.4542,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 0.22147627174854279,
-      "rewards/margins": 1.249265193939209,
-      "rewards/rejected": -1.0277888774871826,
-      "step": 1270
-    },
-    {
-      "epoch": 1.32,
-      "learning_rate": 3.109709480122324e-07,
-      "logits/chosen": -2.5518264770507812,
-      "logits/rejected": -2.5967154502868652,
-      "logps/chosen": -294.47418212890625,
-      "logps/rejected": -272.46380615234375,
-      "loss": 0.4535,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.15300384163856506,
-      "rewards/margins": 0.9860183596611023,
-      "rewards/rejected": -0.8330146074295044,
-      "step": 1280
-    },
-    {
-      "epoch": 1.33,
-      "learning_rate": 3.0905963302752294e-07,
-      "logits/chosen": -2.5392801761627197,
-      "logits/rejected": -2.513974666595459,
-      "logps/chosen": -308.92633056640625,
-      "logps/rejected": -276.67645263671875,
-      "loss": 0.4468,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.11630193144083023,
-      "rewards/margins": 1.0185792446136475,
-      "rewards/rejected": -0.9022773504257202,
-      "step": 1290
-    },
-    {
-      "epoch": 1.34,
-      "learning_rate": 3.071483180428134e-07,
-      "logits/chosen": -2.665621757507324,
-      "logits/rejected": -2.6524558067321777,
-      "logps/chosen": -398.4109802246094,
-      "logps/rejected": -275.1439514160156,
-      "loss": 0.446,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.23084072768688202,
-      "rewards/margins": 0.8804351091384888,
-      "rewards/rejected": -0.6495944261550903,
-      "step": 1300
-    },
-    {
-      "epoch": 1.34,
-      "eval_logits/chosen": -2.727848768234253,
-      "eval_logits/rejected": -2.6739702224731445,
-      "eval_logps/chosen": -367.76025390625,
-      "eval_logps/rejected": -292.675048828125,
-      "eval_loss": 0.5186832547187805,
-      "eval_rewards/accuracies": 0.7420634627342224,
-      "eval_rewards/chosen": 0.20951056480407715,
-      "eval_rewards/margins": 0.9057956337928772,
-      "eval_rewards/rejected": -0.6962851285934448,
-      "eval_runtime": 93.7475,
-      "eval_samples_per_second": 21.334,
-      "eval_steps_per_second": 0.672,
-      "step": 1300
-    },
-    {
-      "epoch": 1.35,
-      "learning_rate": 3.0523700305810395e-07,
-      "logits/chosen": -2.6590070724487305,
-      "logits/rejected": -2.571068286895752,
-      "logps/chosen": -349.6532287597656,
-      "logps/rejected": -309.3611145019531,
-      "loss": 0.4328,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.35963043570518494,
-      "rewards/margins": 1.3134393692016602,
-      "rewards/rejected": -0.9538089632987976,
-      "step": 1310
-    },
-    {
-      "epoch": 1.36,
-      "learning_rate": 3.033256880733945e-07,
-      "logits/chosen": -2.5749783515930176,
-      "logits/rejected": -2.5548958778381348,
-      "logps/chosen": -308.1100769042969,
-      "logps/rejected": -252.40725708007812,
-      "loss": 0.4,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.0827903300523758,
-      "rewards/margins": 0.9593188166618347,
-      "rewards/rejected": -0.8765283823013306,
-      "step": 1320
-    },
-    {
-      "epoch": 1.37,
-      "learning_rate": 3.01414373088685e-07,
-      "logits/chosen": -2.5704667568206787,
-      "logits/rejected": -2.5135350227355957,
-      "logps/chosen": -360.36517333984375,
-      "logps/rejected": -275.2553405761719,
-      "loss": 0.4293,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.13265377283096313,
-      "rewards/margins": 1.178371548652649,
-      "rewards/rejected": -1.0457178354263306,
-      "step": 1330
-    },
-    {
-      "epoch": 1.38,
-      "learning_rate": 2.9950305810397555e-07,
-      "logits/chosen": -2.686350107192993,
-      "logits/rejected": -2.5901427268981934,
-      "logps/chosen": -342.09918212890625,
-      "logps/rejected": -274.7805480957031,
-      "loss": 0.4317,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.21238569915294647,
-      "rewards/margins": 1.2895253896713257,
-      "rewards/rejected": -1.0771398544311523,
-      "step": 1340
-    },
-    {
-      "epoch": 1.39,
-      "learning_rate": 2.9759174311926603e-07,
-      "logits/chosen": -2.6003451347351074,
-      "logits/rejected": -2.5708560943603516,
-      "logps/chosen": -331.8475341796875,
-      "logps/rejected": -273.82281494140625,
-      "loss": 0.4456,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -0.016350137069821358,
-      "rewards/margins": 0.9871395826339722,
-      "rewards/rejected": -1.0034897327423096,
-      "step": 1350
-    },
-    {
-      "epoch": 1.4,
-      "learning_rate": 2.9568042813455656e-07,
-      "logits/chosen": -2.581772804260254,
-      "logits/rejected": -2.5919508934020996,
-      "logps/chosen": -348.9093933105469,
-      "logps/rejected": -245.66946411132812,
-      "loss": 0.4551,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.1543104499578476,
-      "rewards/margins": 1.031022310256958,
-      "rewards/rejected": -0.8767116665840149,
-      "step": 1360
-    },
-    {
-      "epoch": 1.41,
-      "learning_rate": 2.937691131498471e-07,
-      "logits/chosen": -2.6790993213653564,
-      "logits/rejected": -2.57248854637146,
-      "logps/chosen": -364.57208251953125,
-      "logps/rejected": -291.68194580078125,
-      "loss": 0.4674,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.1870456337928772,
-      "rewards/margins": 1.2713440656661987,
-      "rewards/rejected": -1.0842984914779663,
-      "step": 1370
-    },
-    {
-      "epoch": 1.42,
-      "learning_rate": 2.918577981651376e-07,
-      "logits/chosen": -2.595109224319458,
-      "logits/rejected": -2.5366103649139404,
-      "logps/chosen": -301.737548828125,
-      "logps/rejected": -275.08648681640625,
-      "loss": 0.4321,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.10519032180309296,
-      "rewards/margins": 0.8715200424194336,
-      "rewards/rejected": -0.7663297653198242,
-      "step": 1380
-    },
-    {
-      "epoch": 1.43,
-      "learning_rate": 2.8994648318042816e-07,
-      "logits/chosen": -2.621727466583252,
-      "logits/rejected": -2.5707826614379883,
-      "logps/chosen": -336.1698913574219,
-      "logps/rejected": -265.3817443847656,
-      "loss": 0.4192,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.37031644582748413,
-      "rewards/margins": 1.0992939472198486,
-      "rewards/rejected": -0.7289775013923645,
-      "step": 1390
-    },
-    {
-      "epoch": 1.44,
-      "learning_rate": 2.8803516819571863e-07,
-      "logits/chosen": -2.695518970489502,
-      "logits/rejected": -2.606881618499756,
-      "logps/chosen": -396.0542297363281,
-      "logps/rejected": -309.7992248535156,
-      "loss": 0.472,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.2515701651573181,
-      "rewards/margins": 1.056049108505249,
-      "rewards/rejected": -0.8044789433479309,
-      "step": 1400
-    },
-    {
-      "epoch": 1.44,
-      "eval_logits/chosen": -2.7264468669891357,
-      "eval_logits/rejected": -2.671557903289795,
-      "eval_logps/chosen": -367.6226806640625,
-      "eval_logps/rejected": -292.1658630371094,
-      "eval_loss": 0.5153725147247314,
-      "eval_rewards/accuracies": 0.7539682388305664,
-      "eval_rewards/chosen": 0.22327157855033875,
-      "eval_rewards/margins": 0.8686384558677673,
-      "eval_rewards/rejected": -0.645366907119751,
-      "eval_runtime": 93.411,
-      "eval_samples_per_second": 21.411,
-      "eval_steps_per_second": 0.674,
-      "step": 1400
-    },
-    {
-      "epoch": 1.46,
-      "learning_rate": 2.8612385321100917e-07,
-      "logits/chosen": -2.505145311355591,
-      "logits/rejected": -2.462804079055786,
-      "logps/chosen": -304.18524169921875,
-      "logps/rejected": -281.14593505859375,
-      "loss": 0.4304,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.17369304597377777,
-      "rewards/margins": 0.9988886117935181,
-      "rewards/rejected": -0.8251956105232239,
-      "step": 1410
-    },
-    {
-      "epoch": 1.47,
-      "learning_rate": 2.842125382262997e-07,
-      "logits/chosen": -2.5498111248016357,
-      "logits/rejected": -2.4971938133239746,
-      "logps/chosen": -320.25738525390625,
-      "logps/rejected": -273.7497863769531,
-      "loss": 0.4526,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.2570057809352875,
-      "rewards/margins": 1.0560768842697144,
-      "rewards/rejected": -0.799071192741394,
-      "step": 1420
-    },
-    {
-      "epoch": 1.48,
-      "learning_rate": 2.8230122324159023e-07,
-      "logits/chosen": -2.496542453765869,
-      "logits/rejected": -2.508702516555786,
-      "logps/chosen": -366.1961975097656,
-      "logps/rejected": -331.0256652832031,
-      "loss": 0.4079,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 0.43470677733421326,
-      "rewards/margins": 1.43040132522583,
-      "rewards/rejected": -0.9956945180892944,
-      "step": 1430
-    },
-    {
-      "epoch": 1.49,
-      "learning_rate": 2.8038990825688076e-07,
-      "logits/chosen": -2.560516357421875,
-      "logits/rejected": -2.5475456714630127,
-      "logps/chosen": -283.05596923828125,
-      "logps/rejected": -232.9197998046875,
-      "loss": 0.433,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.13742788136005402,
-      "rewards/margins": 0.8214957118034363,
-      "rewards/rejected": -0.6840678453445435,
-      "step": 1440
-    },
-    {
-      "epoch": 1.5,
-      "learning_rate": 2.784785932721712e-07,
-      "logits/chosen": -2.58160662651062,
-      "logits/rejected": -2.504528045654297,
-      "logps/chosen": -342.2316589355469,
-      "logps/rejected": -277.2345275878906,
-      "loss": 0.4591,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.18114377558231354,
-      "rewards/margins": 1.0414986610412598,
-      "rewards/rejected": -0.8603549003601074,
-      "step": 1450
-    },
-    {
-      "epoch": 1.51,
-      "learning_rate": 2.765672782874617e-07,
-      "logits/chosen": -2.579395294189453,
-      "logits/rejected": -2.503072738647461,
-      "logps/chosen": -343.522216796875,
-      "logps/rejected": -258.1620178222656,
-      "loss": 0.4543,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.22307118773460388,
-      "rewards/margins": 0.9056950807571411,
-      "rewards/rejected": -0.6826238632202148,
-      "step": 1460
-    },
-    {
-      "epoch": 1.52,
-      "learning_rate": 2.7465596330275225e-07,
-      "logits/chosen": -2.5702877044677734,
-      "logits/rejected": -2.550537109375,
-      "logps/chosen": -358.40081787109375,
-      "logps/rejected": -223.83908081054688,
-      "loss": 0.4404,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.20269151031970978,
-      "rewards/margins": 1.1862519979476929,
-      "rewards/rejected": -0.9835604429244995,
-      "step": 1470
-    },
-    {
-      "epoch": 1.53,
-      "learning_rate": 2.727446483180428e-07,
-      "logits/chosen": -2.630905866622925,
-      "logits/rejected": -2.5621542930603027,
-      "logps/chosen": -324.4598083496094,
-      "logps/rejected": -284.69232177734375,
-      "loss": 0.435,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.05295666307210922,
-      "rewards/margins": 0.926548182964325,
-      "rewards/rejected": -0.8735915422439575,
-      "step": 1480
-    },
-    {
-      "epoch": 1.54,
-      "learning_rate": 2.708333333333333e-07,
-      "logits/chosen": -2.4956183433532715,
-      "logits/rejected": -2.499742031097412,
-      "logps/chosen": -307.091552734375,
-      "logps/rejected": -282.85186767578125,
-      "loss": 0.4138,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.10083981603384018,
-      "rewards/margins": 1.0654906034469604,
-      "rewards/rejected": -0.9646507501602173,
-      "step": 1490
-    },
-    {
-      "epoch": 1.55,
-      "learning_rate": 2.6892201834862385e-07,
-      "logits/chosen": -2.535083770751953,
-      "logits/rejected": -2.5149922370910645,
-      "logps/chosen": -342.21600341796875,
-      "logps/rejected": -258.53076171875,
-      "loss": 0.4425,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.20716246962547302,
-      "rewards/margins": 1.2411352396011353,
-      "rewards/rejected": -1.033972978591919,
-      "step": 1500
-    },
-    {
-      "epoch": 1.55,
-      "eval_logits/chosen": -2.724445104598999,
-      "eval_logits/rejected": -2.6694822311401367,
-      "eval_logps/chosen": -367.8693542480469,
-      "eval_logps/rejected": -292.79150390625,
-      "eval_loss": 0.5157531499862671,
-      "eval_rewards/accuracies": 0.738095223903656,
-      "eval_rewards/chosen": 0.19860157370567322,
-      "eval_rewards/margins": 0.9065337777137756,
-      "eval_rewards/rejected": -0.7079322934150696,
-      "eval_runtime": 93.8811,
-      "eval_samples_per_second": 21.304,
-      "eval_steps_per_second": 0.671,
-      "step": 1500
-    },
-    {
-      "epoch": 1.56,
-      "learning_rate": 2.6701070336391433e-07,
-      "logits/chosen": -2.611865282058716,
-      "logits/rejected": -2.6318869590759277,
-      "logps/chosen": -405.0105895996094,
-      "logps/rejected": -297.174560546875,
-      "loss": 0.4283,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 0.23556602001190186,
-      "rewards/margins": 0.9569095373153687,
-      "rewards/rejected": -0.7213433980941772,
-      "step": 1510
-    },
-    {
-      "epoch": 1.57,
-      "learning_rate": 2.6509938837920486e-07,
-      "logits/chosen": -2.6025757789611816,
-      "logits/rejected": -2.5256543159484863,
-      "logps/chosen": -291.0908508300781,
-      "logps/rejected": -270.0452575683594,
-      "loss": 0.4221,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 0.2756959795951843,
-      "rewards/margins": 1.0691989660263062,
-      "rewards/rejected": -0.793502926826477,
-      "step": 1520
-    },
-    {
-      "epoch": 1.58,
-      "learning_rate": 2.631880733944954e-07,
-      "logits/chosen": -2.6069562435150146,
-      "logits/rejected": -2.571159839630127,
-      "logps/chosen": -325.5701599121094,
-      "logps/rejected": -268.36822509765625,
-      "loss": 0.4109,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.26621466875076294,
-      "rewards/margins": 0.8793444633483887,
-      "rewards/rejected": -0.613129734992981,
-      "step": 1530
-    },
-    {
-      "epoch": 1.59,
-      "learning_rate": 2.612767584097859e-07,
-      "logits/chosen": -2.629546880722046,
-      "logits/rejected": -2.572211742401123,
-      "logps/chosen": -323.9189453125,
-      "logps/rejected": -257.16033935546875,
-      "loss": 0.4332,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 0.28304505348205566,
-      "rewards/margins": 0.9886387586593628,
-      "rewards/rejected": -0.7055937051773071,
-      "step": 1540
-    },
-    {
-      "epoch": 1.6,
-      "learning_rate": 2.5936544342507646e-07,
-      "logits/chosen": -2.647836208343506,
-      "logits/rejected": -2.506730556488037,
-      "logps/chosen": -352.39337158203125,
-      "logps/rejected": -284.34246826171875,
-      "loss": 0.4266,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.32185670733451843,
-      "rewards/margins": 1.156570553779602,
-      "rewards/rejected": -0.8347137570381165,
-      "step": 1550
-    },
-    {
-      "epoch": 1.61,
-      "learning_rate": 2.5745412844036693e-07,
-      "logits/chosen": -2.5880885124206543,
-      "logits/rejected": -2.5855612754821777,
-      "logps/chosen": -352.1588134765625,
-      "logps/rejected": -263.62005615234375,
-      "loss": 0.4388,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.24963721632957458,
-      "rewards/margins": 1.1166155338287354,
-      "rewards/rejected": -0.8669784665107727,
-      "step": 1560
-    },
-    {
-      "epoch": 1.62,
-      "learning_rate": 2.5554281345565747e-07,
-      "logits/chosen": -2.6382758617401123,
-      "logits/rejected": -2.5842955112457275,
-      "logps/chosen": -328.3256530761719,
-      "logps/rejected": -290.60968017578125,
-      "loss": 0.4219,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.29845088720321655,
-      "rewards/margins": 1.0773983001708984,
-      "rewards/rejected": -0.7789472937583923,
-      "step": 1570
-    },
-    {
-      "epoch": 1.63,
-      "learning_rate": 2.53631498470948e-07,
-      "logits/chosen": -2.6656248569488525,
-      "logits/rejected": -2.6021933555603027,
-      "logps/chosen": -358.393310546875,
-      "logps/rejected": -268.1978454589844,
-      "loss": 0.3933,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.34928596019744873,
-      "rewards/margins": 1.214766263961792,
-      "rewards/rejected": -0.8654803037643433,
-      "step": 1580
-    },
-    {
-      "epoch": 1.64,
-      "learning_rate": 2.5172018348623853e-07,
-      "logits/chosen": -2.5791847705841064,
-      "logits/rejected": -2.5275607109069824,
-      "logps/chosen": -348.0746765136719,
-      "logps/rejected": -269.4404602050781,
-      "loss": 0.4482,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.1630961000919342,
-      "rewards/margins": 1.0848236083984375,
-      "rewards/rejected": -0.9217275381088257,
-      "step": 1590
-    },
-    {
-      "epoch": 1.65,
-      "learning_rate": 2.4980886850152906e-07,
-      "logits/chosen": -2.624077796936035,
-      "logits/rejected": -2.550253391265869,
-      "logps/chosen": -353.8025817871094,
-      "logps/rejected": -263.6338806152344,
-      "loss": 0.434,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.2791728377342224,
-      "rewards/margins": 1.2417490482330322,
-      "rewards/rejected": -0.9625762104988098,
-      "step": 1600
-    },
-    {
-      "epoch": 1.65,
-      "eval_logits/chosen": -2.718714475631714,
-      "eval_logits/rejected": -2.6639087200164795,
-      "eval_logps/chosen": -367.8188171386719,
-      "eval_logps/rejected": -292.5534973144531,
-      "eval_loss": 0.5148204565048218,
-      "eval_rewards/accuracies": 0.738095223903656,
-      "eval_rewards/chosen": 0.20365335047245026,
-      "eval_rewards/margins": 0.8877854347229004,
-      "eval_rewards/rejected": -0.6841320395469666,
-      "eval_runtime": 93.9834,
-      "eval_samples_per_second": 21.28,
-      "eval_steps_per_second": 0.67,
-      "step": 1600
-    },
-    {
-      "epoch": 1.66,
-      "learning_rate": 2.478975535168196e-07,
-      "logits/chosen": -2.595499038696289,
-      "logits/rejected": -2.548311948776245,
-      "logps/chosen": -330.22088623046875,
-      "logps/rejected": -250.4892120361328,
-      "loss": 0.4271,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.27131223678588867,
-      "rewards/margins": 1.1109501123428345,
-      "rewards/rejected": -0.8396379351615906,
-      "step": 1610
-    },
-    {
-      "epoch": 1.67,
-      "learning_rate": 2.459862385321101e-07,
-      "logits/chosen": -2.7138476371765137,
-      "logits/rejected": -2.624547243118286,
-      "logps/chosen": -395.07122802734375,
-      "logps/rejected": -291.44482421875,
-      "loss": 0.4514,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.198801189661026,
-      "rewards/margins": 0.8508071899414062,
-      "rewards/rejected": -0.6520059108734131,
-      "step": 1620
-    },
-    {
-      "epoch": 1.68,
-      "learning_rate": 2.440749235474006e-07,
-      "logits/chosen": -2.651254653930664,
-      "logits/rejected": -2.5482094287872314,
-      "logps/chosen": -360.50970458984375,
-      "logps/rejected": -281.64727783203125,
-      "loss": 0.4303,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.36835628747940063,
-      "rewards/margins": 1.4067318439483643,
-      "rewards/rejected": -1.0383756160736084,
-      "step": 1630
-    },
-    {
-      "epoch": 1.69,
-      "learning_rate": 2.421636085626911e-07,
-      "logits/chosen": -2.5148165225982666,
-      "logits/rejected": -2.4600250720977783,
-      "logps/chosen": -334.34246826171875,
-      "logps/rejected": -256.6380310058594,
-      "loss": 0.4345,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.3089974820613861,
-      "rewards/margins": 0.9867910146713257,
-      "rewards/rejected": -0.6777936220169067,
-      "step": 1640
-    },
-    {
-      "epoch": 1.7,
-      "learning_rate": 2.402522935779816e-07,
-      "logits/chosen": -2.645481586456299,
-      "logits/rejected": -2.6082756519317627,
-      "logps/chosen": -361.30474853515625,
-      "logps/rejected": -317.3288879394531,
-      "loss": 0.4247,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 0.28494271636009216,
-      "rewards/margins": 1.162066102027893,
-      "rewards/rejected": -0.8771233558654785,
-      "step": 1650
-    },
-    {
-      "epoch": 1.71,
-      "learning_rate": 2.3834097859327215e-07,
-      "logits/chosen": -2.5973002910614014,
-      "logits/rejected": -2.552525281906128,
-      "logps/chosen": -309.293701171875,
-      "logps/rejected": -275.47344970703125,
-      "loss": 0.438,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.2845771014690399,
-      "rewards/margins": 1.1519324779510498,
-      "rewards/rejected": -0.8673553466796875,
-      "step": 1660
-    },
-    {
-      "epoch": 1.72,
-      "learning_rate": 2.3642966360856268e-07,
-      "logits/chosen": -2.644890546798706,
-      "logits/rejected": -2.6000285148620605,
-      "logps/chosen": -344.3207702636719,
-      "logps/rejected": -269.1863708496094,
-      "loss": 0.3958,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.44529637694358826,
-      "rewards/margins": 1.219225525856018,
-      "rewards/rejected": -0.7739290595054626,
-      "step": 1670
-    },
-    {
-      "epoch": 1.73,
-      "learning_rate": 2.345183486238532e-07,
-      "logits/chosen": -2.603766918182373,
-      "logits/rejected": -2.548722743988037,
-      "logps/chosen": -322.1404724121094,
-      "logps/rejected": -316.88507080078125,
-      "loss": 0.424,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.1287519633769989,
-      "rewards/margins": 0.9171475172042847,
-      "rewards/rejected": -0.7883955836296082,
-      "step": 1680
-    },
-    {
-      "epoch": 1.74,
-      "learning_rate": 2.3260703363914372e-07,
-      "logits/chosen": -2.6459596157073975,
-      "logits/rejected": -2.615452766418457,
-      "logps/chosen": -381.41748046875,
-      "logps/rejected": -300.71905517578125,
-      "loss": 0.4087,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.216604083776474,
-      "rewards/margins": 1.141821026802063,
-      "rewards/rejected": -0.9252168536186218,
-      "step": 1690
-    },
-    {
-      "epoch": 1.75,
-      "learning_rate": 2.3069571865443425e-07,
-      "logits/chosen": -2.63240122795105,
-      "logits/rejected": -2.6180896759033203,
-      "logps/chosen": -291.97161865234375,
-      "logps/rejected": -283.84661865234375,
-      "loss": 0.4209,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.04391371086239815,
-      "rewards/margins": 0.8607124090194702,
-      "rewards/rejected": -0.9046260714530945,
-      "step": 1700
-    },
-    {
-      "epoch": 1.75,
-      "eval_logits/chosen": -2.718451976776123,
-      "eval_logits/rejected": -2.6635725498199463,
-      "eval_logps/chosen": -368.5582275390625,
-      "eval_logps/rejected": -293.53076171875,
-      "eval_loss": 0.5146352052688599,
-      "eval_rewards/accuracies": 0.7460317611694336,
-      "eval_rewards/chosen": 0.1297152191400528,
-      "eval_rewards/margins": 0.9115698337554932,
-      "eval_rewards/rejected": -0.781854510307312,
-      "eval_runtime": 93.2472,
-      "eval_samples_per_second": 21.448,
-      "eval_steps_per_second": 0.676,
-      "step": 1700
-    },
-    {
-      "epoch": 1.76,
-      "learning_rate": 2.2878440366972476e-07,
-      "logits/chosen": -2.6936182975769043,
-      "logits/rejected": -2.6290268898010254,
-      "logps/chosen": -355.96124267578125,
-      "logps/rejected": -341.81549072265625,
-      "loss": 0.4177,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.23748740553855896,
-      "rewards/margins": 1.278755784034729,
-      "rewards/rejected": -1.0412683486938477,
-      "step": 1710
-    },
-    {
-      "epoch": 1.78,
-      "learning_rate": 2.268730886850153e-07,
-      "logits/chosen": -2.5507352352142334,
-      "logits/rejected": -2.453554391860962,
-      "logps/chosen": -344.82452392578125,
-      "logps/rejected": -273.4502868652344,
-      "loss": 0.4313,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.22865517437458038,
-      "rewards/margins": 0.964372992515564,
-      "rewards/rejected": -0.7357178330421448,
-      "step": 1720
-    },
-    {
-      "epoch": 1.79,
-      "learning_rate": 2.249617737003058e-07,
-      "logits/chosen": -2.6256864070892334,
-      "logits/rejected": -2.5964672565460205,
-      "logps/chosen": -369.0529479980469,
-      "logps/rejected": -331.5936279296875,
-      "loss": 0.4216,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.12068464607000351,
-      "rewards/margins": 1.1605781316757202,
-      "rewards/rejected": -1.039893388748169,
-      "step": 1730
-    },
-    {
-      "epoch": 1.8,
-      "learning_rate": 2.2305045871559633e-07,
-      "logits/chosen": -2.565049171447754,
-      "logits/rejected": -2.510228157043457,
-      "logps/chosen": -359.94671630859375,
-      "logps/rejected": -297.53460693359375,
-      "loss": 0.4199,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.034449171274900436,
-      "rewards/margins": 0.8863730430603027,
-      "rewards/rejected": -0.920822262763977,
-      "step": 1740
-    },
-    {
-      "epoch": 1.81,
-      "learning_rate": 2.2113914373088686e-07,
-      "logits/chosen": -2.656843662261963,
-      "logits/rejected": -2.610401153564453,
-      "logps/chosen": -363.86944580078125,
-      "logps/rejected": -314.6811218261719,
-      "loss": 0.4216,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 0.2026744782924652,
-      "rewards/margins": 1.1955655813217163,
-      "rewards/rejected": -0.9928911328315735,
-      "step": 1750
-    },
-    {
-      "epoch": 1.82,
-      "learning_rate": 2.1922782874617736e-07,
-      "logits/chosen": -2.5376534461975098,
-      "logits/rejected": -2.570894956588745,
-      "logps/chosen": -300.3117980957031,
-      "logps/rejected": -328.97003173828125,
-      "loss": 0.4272,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.10297539085149765,
-      "rewards/margins": 1.1623529195785522,
-      "rewards/rejected": -1.0593774318695068,
-      "step": 1760
-    },
-    {
-      "epoch": 1.83,
-      "learning_rate": 2.1731651376146787e-07,
-      "logits/chosen": -2.545215129852295,
-      "logits/rejected": -2.538797616958618,
-      "logps/chosen": -290.8783874511719,
-      "logps/rejected": -284.0111999511719,
-      "loss": 0.446,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.048707880079746246,
-      "rewards/margins": 0.9897578358650208,
-      "rewards/rejected": -0.9410500526428223,
-      "step": 1770
-    },
-    {
-      "epoch": 1.84,
-      "learning_rate": 2.154051987767584e-07,
-      "logits/chosen": -2.6110103130340576,
-      "logits/rejected": -2.5820937156677246,
-      "logps/chosen": -335.0147705078125,
-      "logps/rejected": -273.63763427734375,
-      "loss": 0.4322,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.0762074887752533,
-      "rewards/margins": 1.0348106622695923,
-      "rewards/rejected": -0.9586030840873718,
-      "step": 1780
-    },
-    {
-      "epoch": 1.85,
-      "learning_rate": 2.134938837920489e-07,
-      "logits/chosen": -2.545682430267334,
-      "logits/rejected": -2.560351848602295,
-      "logps/chosen": -249.69970703125,
-      "logps/rejected": -240.6475067138672,
-      "loss": 0.4126,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.14385946094989777,
-      "rewards/margins": 1.0989620685577393,
-      "rewards/rejected": -0.9551024436950684,
-      "step": 1790
-    },
-    {
-      "epoch": 1.86,
-      "learning_rate": 2.1158256880733944e-07,
-      "logits/chosen": -2.6015000343322754,
-      "logits/rejected": -2.561143636703491,
-      "logps/chosen": -373.9018859863281,
-      "logps/rejected": -261.9052429199219,
-      "loss": 0.4128,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.21687176823616028,
-      "rewards/margins": 1.2524230480194092,
-      "rewards/rejected": -1.0355513095855713,
-      "step": 1800
-    },
-    {
-      "epoch": 1.86,
-      "eval_logits/chosen": -2.7194206714630127,
-      "eval_logits/rejected": -2.665125608444214,
-      "eval_logps/chosen": -368.43719482421875,
-      "eval_logps/rejected": -293.5338439941406,
-      "eval_loss": 0.5128700137138367,
-      "eval_rewards/accuracies": 0.738095223903656,
-      "eval_rewards/chosen": 0.14181897044181824,
-      "eval_rewards/margins": 0.9239831566810608,
-      "eval_rewards/rejected": -0.7821642160415649,
-      "eval_runtime": 93.6913,
-      "eval_samples_per_second": 21.347,
-      "eval_steps_per_second": 0.672,
-      "step": 1800
-    },
-    {
-      "epoch": 1.87,
-      "learning_rate": 2.0967125382262994e-07,
-      "logits/chosen": -2.5647854804992676,
-      "logits/rejected": -2.549739360809326,
-      "logps/chosen": -345.7157287597656,
-      "logps/rejected": -280.04290771484375,
-      "loss": 0.4209,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.2801140248775482,
-      "rewards/margins": 1.1526819467544556,
-      "rewards/rejected": -0.8725678324699402,
-      "step": 1810
-    },
-    {
-      "epoch": 1.88,
-      "learning_rate": 2.0775993883792048e-07,
-      "logits/chosen": -2.5792086124420166,
-      "logits/rejected": -2.5236520767211914,
-      "logps/chosen": -358.15264892578125,
-      "logps/rejected": -264.4053955078125,
-      "loss": 0.4224,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.24478964507579803,
-      "rewards/margins": 1.050995111465454,
-      "rewards/rejected": -0.8062052726745605,
-      "step": 1820
-    },
-    {
-      "epoch": 1.89,
-      "learning_rate": 2.05848623853211e-07,
-      "logits/chosen": -2.627408504486084,
-      "logits/rejected": -2.6247506141662598,
-      "logps/chosen": -375.7535400390625,
-      "logps/rejected": -278.0137023925781,
-      "loss": 0.4119,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.061488620936870575,
-      "rewards/margins": 1.0074807405471802,
-      "rewards/rejected": -0.945992112159729,
-      "step": 1830
-    },
-    {
-      "epoch": 1.9,
-      "learning_rate": 2.0393730886850151e-07,
-      "logits/chosen": -2.6022982597351074,
-      "logits/rejected": -2.514789581298828,
-      "logps/chosen": -372.36346435546875,
-      "logps/rejected": -270.7804260253906,
-      "loss": 0.4066,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.11869986355304718,
-      "rewards/margins": 1.225434422492981,
-      "rewards/rejected": -1.1067346334457397,
-      "step": 1840
-    },
-    {
-      "epoch": 1.91,
-      "learning_rate": 2.0202599388379205e-07,
-      "logits/chosen": -2.643916130065918,
-      "logits/rejected": -2.6036553382873535,
-      "logps/chosen": -346.41107177734375,
-      "logps/rejected": -342.21392822265625,
-      "loss": 0.3963,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.09456288814544678,
-      "rewards/margins": 1.0665562152862549,
-      "rewards/rejected": -0.9719934463500977,
-      "step": 1850
-    },
-    {
-      "epoch": 1.92,
-      "learning_rate": 2.0011467889908258e-07,
-      "logits/chosen": -2.6812074184417725,
-      "logits/rejected": -2.5795161724090576,
-      "logps/chosen": -375.649658203125,
-      "logps/rejected": -224.4620361328125,
-      "loss": 0.4207,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.3432249426841736,
-      "rewards/margins": 1.4275915622711182,
-      "rewards/rejected": -1.0843665599822998,
-      "step": 1860
-    },
-    {
-      "epoch": 1.93,
-      "learning_rate": 1.9820336391437308e-07,
-      "logits/chosen": -2.569981575012207,
-      "logits/rejected": -2.5870277881622314,
-      "logps/chosen": -320.9058837890625,
-      "logps/rejected": -265.43145751953125,
-      "loss": 0.4248,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.09469388425350189,
-      "rewards/margins": 0.9917991757392883,
-      "rewards/rejected": -0.89710533618927,
-      "step": 1870
-    },
-    {
-      "epoch": 1.94,
-      "learning_rate": 1.9629204892966362e-07,
-      "logits/chosen": -2.670983076095581,
-      "logits/rejected": -2.632185459136963,
-      "logps/chosen": -357.2303466796875,
-      "logps/rejected": -270.21966552734375,
-      "loss": 0.4084,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.3034873604774475,
-      "rewards/margins": 1.3726078271865845,
-      "rewards/rejected": -1.0691205263137817,
-      "step": 1880
-    },
-    {
-      "epoch": 1.95,
-      "learning_rate": 1.943807339449541e-07,
-      "logits/chosen": -2.5820472240448,
-      "logits/rejected": -2.480771780014038,
-      "logps/chosen": -325.0736389160156,
-      "logps/rejected": -292.05291748046875,
-      "loss": 0.4147,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.13287527859210968,
-      "rewards/margins": 1.379725456237793,
-      "rewards/rejected": -1.2468502521514893,
-      "step": 1890
-    },
-    {
-      "epoch": 1.96,
-      "learning_rate": 1.9246941896024463e-07,
-      "logits/chosen": -2.5788347721099854,
-      "logits/rejected": -2.6083314418792725,
-      "logps/chosen": -331.9407958984375,
-      "logps/rejected": -301.12176513671875,
-      "loss": 0.4685,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.006614005658775568,
-      "rewards/margins": 0.950186550617218,
-      "rewards/rejected": -0.9435726404190063,
-      "step": 1900
-    },
-    {
-      "epoch": 1.96,
-      "eval_logits/chosen": -2.7248175144195557,
-      "eval_logits/rejected": -2.6709485054016113,
-      "eval_logps/chosen": -368.8879089355469,
-      "eval_logps/rejected": -293.96771240234375,
-      "eval_loss": 0.5125181078910828,
-      "eval_rewards/accuracies": 0.7420634627342224,
-      "eval_rewards/chosen": 0.096745066344738,
-      "eval_rewards/margins": 0.9222998023033142,
-      "eval_rewards/rejected": -0.8255547881126404,
-      "eval_runtime": 93.5081,
-      "eval_samples_per_second": 21.389,
-      "eval_steps_per_second": 0.674,
-      "step": 1900
-    },
-    {
-      "epoch": 1.97,
-      "learning_rate": 1.9055810397553516e-07,
-      "logits/chosen": -2.6049692630767822,
-      "logits/rejected": -2.5768351554870605,
-      "logps/chosen": -335.76861572265625,
-      "logps/rejected": -283.60418701171875,
-      "loss": 0.4452,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.1541665941476822,
-      "rewards/margins": 1.058864951133728,
-      "rewards/rejected": -0.904698371887207,
-      "step": 1910
-    },
-    {
-      "epoch": 1.98,
-      "learning_rate": 1.8864678899082566e-07,
-      "logits/chosen": -2.58392333984375,
-      "logits/rejected": -2.5964534282684326,
-      "logps/chosen": -334.490966796875,
-      "logps/rejected": -253.0091552734375,
-      "loss": 0.4227,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.1425321400165558,
-      "rewards/margins": 1.186306357383728,
-      "rewards/rejected": -1.0437742471694946,
-      "step": 1920
-    },
-    {
-      "epoch": 1.99,
-      "learning_rate": 1.867354740061162e-07,
-      "logits/chosen": -2.5509891510009766,
-      "logits/rejected": -2.574993371963501,
-      "logps/chosen": -318.6093444824219,
-      "logps/rejected": -283.29803466796875,
-      "loss": 0.4151,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.036455459892749786,
-      "rewards/margins": 1.2164264917373657,
-      "rewards/rejected": -1.1799709796905518,
-      "step": 1930
-    },
-    {
-      "epoch": 2.0,
-      "learning_rate": 1.8482415902140673e-07,
-      "logits/chosen": -2.606382131576538,
-      "logits/rejected": -2.597195625305176,
-      "logps/chosen": -324.63934326171875,
-      "logps/rejected": -269.2276306152344,
-      "loss": 0.4031,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.2206292450428009,
-      "rewards/margins": 1.2134065628051758,
-      "rewards/rejected": -0.9927773475646973,
-      "step": 1940
-    },
-    {
-      "epoch": 2.01,
-      "learning_rate": 1.8291284403669723e-07,
-      "logits/chosen": -2.585014820098877,
-      "logits/rejected": -2.621039628982544,
-      "logps/chosen": -318.5997314453125,
-      "logps/rejected": -273.28533935546875,
-      "loss": 0.3465,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.2141040861606598,
-      "rewards/margins": 1.1809941530227661,
-      "rewards/rejected": -0.9668900370597839,
-      "step": 1950
-    },
-    {
-      "epoch": 2.02,
-      "learning_rate": 1.8100152905198777e-07,
-      "logits/chosen": -2.4339516162872314,
-      "logits/rejected": -2.442187547683716,
-      "logps/chosen": -335.56097412109375,
-      "logps/rejected": -327.67803955078125,
-      "loss": 0.3465,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.2534107565879822,
-      "rewards/margins": 1.4552501440048218,
-      "rewards/rejected": -1.2018392086029053,
-      "step": 1960
-    },
-    {
-      "epoch": 2.03,
-      "learning_rate": 1.7909021406727827e-07,
-      "logits/chosen": -2.5640928745269775,
-      "logits/rejected": -2.6635804176330566,
-      "logps/chosen": -324.06988525390625,
-      "logps/rejected": -259.5931091308594,
-      "loss": 0.3612,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.24693092703819275,
-      "rewards/margins": 1.3844406604766846,
-      "rewards/rejected": -1.1375097036361694,
-      "step": 1970
-    },
-    {
-      "epoch": 2.04,
-      "learning_rate": 1.771788990825688e-07,
-      "logits/chosen": -2.653184652328491,
-      "logits/rejected": -2.581162929534912,
-      "logps/chosen": -341.0339050292969,
-      "logps/rejected": -292.974853515625,
-      "loss": 0.3735,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.31022703647613525,
-      "rewards/margins": 1.3054556846618652,
-      "rewards/rejected": -0.99522864818573,
-      "step": 1980
-    },
-    {
-      "epoch": 2.05,
-      "learning_rate": 1.7526758409785934e-07,
-      "logits/chosen": -2.681022882461548,
-      "logits/rejected": -2.5809178352355957,
-      "logps/chosen": -386.1690368652344,
-      "logps/rejected": -304.6114807128906,
-      "loss": 0.3469,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.222796231508255,
-      "rewards/margins": 1.5472052097320557,
-      "rewards/rejected": -1.324409008026123,
-      "step": 1990
-    },
-    {
-      "epoch": 2.06,
-      "learning_rate": 1.7335626911314984e-07,
-      "logits/chosen": -2.6000771522521973,
-      "logits/rejected": -2.522247314453125,
-      "logps/chosen": -287.4322509765625,
-      "logps/rejected": -245.08792114257812,
-      "loss": 0.3605,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 0.005565536208450794,
-      "rewards/margins": 1.2278538942337036,
-      "rewards/rejected": -1.2222882509231567,
-      "step": 2000
-    },
-    {
-      "epoch": 2.06,
-      "eval_logits/chosen": -2.721067428588867,
-      "eval_logits/rejected": -2.6688947677612305,
-      "eval_logps/chosen": -369.22808837890625,
-      "eval_logps/rejected": -294.6591491699219,
-      "eval_loss": 0.5129652619361877,
-      "eval_rewards/accuracies": 0.7301587462425232,
-      "eval_rewards/chosen": 0.06272637844085693,
-      "eval_rewards/margins": 0.9574272036552429,
-      "eval_rewards/rejected": -0.8947007060050964,
-      "eval_runtime": 93.5294,
-      "eval_samples_per_second": 21.384,
-      "eval_steps_per_second": 0.674,
-      "step": 2000
-    },
-    {
-      "epoch": 2.07,
-      "learning_rate": 1.7144495412844037e-07,
-      "logits/chosen": -2.6307573318481445,
-      "logits/rejected": -2.566410541534424,
-      "logps/chosen": -383.6712341308594,
-      "logps/rejected": -295.6425476074219,
-      "loss": 0.3426,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.12579414248466492,
-      "rewards/margins": 1.5774627923965454,
-      "rewards/rejected": -1.4516685009002686,
-      "step": 2010
-    },
-    {
-      "epoch": 2.08,
-      "learning_rate": 1.6953363914373088e-07,
-      "logits/chosen": -2.708552837371826,
-      "logits/rejected": -2.5603816509246826,
-      "logps/chosen": -383.33270263671875,
-      "logps/rejected": -278.8658752441406,
-      "loss": 0.3581,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 0.13873082399368286,
-      "rewards/margins": 1.3727829456329346,
-      "rewards/rejected": -1.234052062034607,
-      "step": 2020
-    },
-    {
-      "epoch": 2.09,
-      "learning_rate": 1.6762232415902138e-07,
-      "logits/chosen": -2.590285539627075,
-      "logits/rejected": -2.574758768081665,
-      "logps/chosen": -320.2268371582031,
-      "logps/rejected": -267.88092041015625,
-      "loss": 0.3694,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.0574222207069397,
-      "rewards/margins": 1.3185408115386963,
-      "rewards/rejected": -1.2611186504364014,
-      "step": 2030
-    },
-    {
-      "epoch": 2.11,
-      "learning_rate": 1.6571100917431192e-07,
-      "logits/chosen": -2.6858019828796387,
-      "logits/rejected": -2.6346988677978516,
-      "logps/chosen": -340.21624755859375,
-      "logps/rejected": -254.05859375,
-      "loss": 0.3503,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.2275957614183426,
-      "rewards/margins": 1.4201277494430542,
-      "rewards/rejected": -1.1925320625305176,
-      "step": 2040
-    },
-    {
-      "epoch": 2.12,
-      "learning_rate": 1.6379969418960242e-07,
-      "logits/chosen": -2.6139819622039795,
-      "logits/rejected": -2.611030101776123,
-      "logps/chosen": -396.6602783203125,
-      "logps/rejected": -331.1925048828125,
-      "loss": 0.3537,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.23956382274627686,
-      "rewards/margins": 1.449977159500122,
-      "rewards/rejected": -1.2104132175445557,
-      "step": 2050
-    },
-    {
-      "epoch": 2.13,
-      "learning_rate": 1.6188837920489295e-07,
-      "logits/chosen": -2.6991240978240967,
-      "logits/rejected": -2.5340638160705566,
-      "logps/chosen": -367.3908386230469,
-      "logps/rejected": -262.90155029296875,
-      "loss": 0.3552,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.16707102954387665,
-      "rewards/margins": 1.3888185024261475,
-      "rewards/rejected": -1.2217473983764648,
-      "step": 2060
-    },
-    {
-      "epoch": 2.14,
-      "learning_rate": 1.5997706422018349e-07,
-      "logits/chosen": -2.729551315307617,
-      "logits/rejected": -2.634174346923828,
-      "logps/chosen": -347.735107421875,
-      "logps/rejected": -352.8389892578125,
-      "loss": 0.3628,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.3393300473690033,
-      "rewards/margins": 1.329689860343933,
-      "rewards/rejected": -0.990359902381897,
-      "step": 2070
-    },
-    {
-      "epoch": 2.15,
-      "learning_rate": 1.58065749235474e-07,
-      "logits/chosen": -2.6678922176361084,
-      "logits/rejected": -2.5802249908447266,
-      "logps/chosen": -405.42071533203125,
-      "logps/rejected": -323.5943298339844,
-      "loss": 0.363,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.4442678391933441,
-      "rewards/margins": 1.3085780143737793,
-      "rewards/rejected": -0.8643101453781128,
-      "step": 2080
-    },
-    {
-      "epoch": 2.16,
-      "learning_rate": 1.5615443425076452e-07,
-      "logits/chosen": -2.5303499698638916,
-      "logits/rejected": -2.5398049354553223,
-      "logps/chosen": -333.7301025390625,
-      "logps/rejected": -273.6751403808594,
-      "loss": 0.3423,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.2193554937839508,
-      "rewards/margins": 1.4085685014724731,
-      "rewards/rejected": -1.1892130374908447,
-      "step": 2090
-    },
-    {
-      "epoch": 2.17,
-      "learning_rate": 1.5424311926605506e-07,
-      "logits/chosen": -2.5035347938537598,
-      "logits/rejected": -2.4870362281799316,
-      "logps/chosen": -297.1385498046875,
-      "logps/rejected": -234.04006958007812,
-      "loss": 0.3463,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.21916255354881287,
-      "rewards/margins": 1.5722987651824951,
-      "rewards/rejected": -1.3531363010406494,
-      "step": 2100
-    },
-    {
-      "epoch": 2.17,
-      "eval_logits/chosen": -2.7217917442321777,
-      "eval_logits/rejected": -2.6708853244781494,
-      "eval_logps/chosen": -369.4024963378906,
-      "eval_logps/rejected": -295.1770324707031,
-      "eval_loss": 0.5123242139816284,
-      "eval_rewards/accuracies": 0.7420634627342224,
-      "eval_rewards/chosen": 0.045285556465387344,
-      "eval_rewards/margins": 0.9917705059051514,
-      "eval_rewards/rejected": -0.9464850425720215,
-      "eval_runtime": 93.4525,
-      "eval_samples_per_second": 21.401,
-      "eval_steps_per_second": 0.674,
-      "step": 2100
-    },
-    {
-      "epoch": 2.18,
-      "learning_rate": 1.5233180428134556e-07,
-      "logits/chosen": -2.6595582962036133,
-      "logits/rejected": -2.6285529136657715,
-      "logps/chosen": -347.6577453613281,
-      "logps/rejected": -276.10015869140625,
-      "loss": 0.347,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.2700585722923279,
-      "rewards/margins": 1.5010900497436523,
-      "rewards/rejected": -1.2310314178466797,
-      "step": 2110
-    },
-    {
-      "epoch": 2.19,
-      "learning_rate": 1.504204892966361e-07,
-      "logits/chosen": -2.5997138023376465,
-      "logits/rejected": -2.5408456325531006,
-      "logps/chosen": -377.7714538574219,
-      "logps/rejected": -303.34222412109375,
-      "loss": 0.35,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.1600765436887741,
-      "rewards/margins": 1.3487540483474731,
-      "rewards/rejected": -1.1886775493621826,
-      "step": 2120
-    },
-    {
-      "epoch": 2.2,
-      "learning_rate": 1.485091743119266e-07,
-      "logits/chosen": -2.7113072872161865,
-      "logits/rejected": -2.543774127960205,
-      "logps/chosen": -316.48150634765625,
-      "logps/rejected": -299.80230712890625,
-      "loss": 0.3588,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.09655191749334335,
-      "rewards/margins": 1.3552639484405518,
-      "rewards/rejected": -1.2587120532989502,
-      "step": 2130
-    },
-    {
-      "epoch": 2.21,
-      "learning_rate": 1.465978593272171e-07,
-      "logits/chosen": -2.590290069580078,
-      "logits/rejected": -2.5230295658111572,
-      "logps/chosen": -252.84683227539062,
-      "logps/rejected": -184.59024047851562,
-      "loss": 0.3436,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.08013294637203217,
-      "rewards/margins": 1.3275083303451538,
-      "rewards/rejected": -1.2473753690719604,
-      "step": 2140
-    },
-    {
-      "epoch": 2.22,
-      "learning_rate": 1.4468654434250764e-07,
-      "logits/chosen": -2.646439790725708,
-      "logits/rejected": -2.597733736038208,
-      "logps/chosen": -399.18341064453125,
-      "logps/rejected": -342.6891174316406,
-      "loss": 0.3589,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.27550849318504333,
-      "rewards/margins": 1.4095003604888916,
-      "rewards/rejected": -1.133992075920105,
-      "step": 2150
-    },
-    {
-      "epoch": 2.23,
-      "learning_rate": 1.4277522935779814e-07,
-      "logits/chosen": -2.5901618003845215,
-      "logits/rejected": -2.635596990585327,
-      "logps/chosen": -347.9039001464844,
-      "logps/rejected": -300.8187255859375,
-      "loss": 0.3482,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/chosen": 0.2792564630508423,
-      "rewards/margins": 1.5975362062454224,
-      "rewards/rejected": -1.318279504776001,
-      "step": 2160
-    },
-    {
-      "epoch": 2.24,
-      "learning_rate": 1.4086391437308867e-07,
-      "logits/chosen": -2.662043333053589,
-      "logits/rejected": -2.637568712234497,
-      "logps/chosen": -365.7566223144531,
-      "logps/rejected": -337.00518798828125,
-      "loss": 0.3335,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.11271584033966064,
-      "rewards/margins": 1.511805534362793,
-      "rewards/rejected": -1.3990895748138428,
-      "step": 2170
-    },
-    {
-      "epoch": 2.25,
-      "learning_rate": 1.389525993883792e-07,
-      "logits/chosen": -2.6112186908721924,
-      "logits/rejected": -2.5363845825195312,
-      "logps/chosen": -290.4798583984375,
-      "logps/rejected": -244.0265350341797,
-      "loss": 0.3783,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.04764432832598686,
-      "rewards/margins": 1.378379225730896,
-      "rewards/rejected": -1.3307349681854248,
-      "step": 2180
-    },
-    {
-      "epoch": 2.26,
-      "learning_rate": 1.370412844036697e-07,
-      "logits/chosen": -2.583376407623291,
-      "logits/rejected": -2.6340744495391846,
-      "logps/chosen": -318.22344970703125,
-      "logps/rejected": -290.05194091796875,
-      "loss": 0.3579,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.15198606252670288,
-      "rewards/margins": 1.2958182096481323,
-      "rewards/rejected": -1.1438322067260742,
-      "step": 2190
-    },
-    {
-      "epoch": 2.27,
-      "learning_rate": 1.3512996941896024e-07,
-      "logits/chosen": -2.5554592609405518,
-      "logits/rejected": -2.5260140895843506,
-      "logps/chosen": -280.261962890625,
-      "logps/rejected": -238.7654571533203,
-      "loss": 0.362,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -0.10016796737909317,
-      "rewards/margins": 1.0856527090072632,
-      "rewards/rejected": -1.1858208179473877,
-      "step": 2200
-    },
-    {
-      "epoch": 2.27,
-      "eval_logits/chosen": -2.7140095233917236,
-      "eval_logits/rejected": -2.662822723388672,
-      "eval_logps/chosen": -369.68109130859375,
-      "eval_logps/rejected": -295.4860534667969,
-      "eval_loss": 0.512495756149292,
-      "eval_rewards/accuracies": 0.738095223903656,
-      "eval_rewards/chosen": 0.017426857724785805,
-      "eval_rewards/margins": 0.994812548160553,
-      "eval_rewards/rejected": -0.9773856401443481,
-      "eval_runtime": 93.8775,
-      "eval_samples_per_second": 21.304,
-      "eval_steps_per_second": 0.671,
-      "step": 2200
-    },
-    {
-      "epoch": 2.28,
-      "learning_rate": 1.3321865443425075e-07,
-      "logits/chosen": -2.5705647468566895,
-      "logits/rejected": -2.408140182495117,
-      "logps/chosen": -349.8976135253906,
-      "logps/rejected": -282.80047607421875,
-      "loss": 0.3624,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.13661615550518036,
-      "rewards/margins": 1.5523512363433838,
-      "rewards/rejected": -1.4157350063323975,
-      "step": 2210
-    },
-    {
-      "epoch": 2.29,
-      "learning_rate": 1.3130733944954128e-07,
-      "logits/chosen": -2.656877279281616,
-      "logits/rejected": -2.573462963104248,
-      "logps/chosen": -363.28594970703125,
-      "logps/rejected": -304.9879455566406,
-      "loss": 0.3338,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.17011162638664246,
-      "rewards/margins": 1.5930747985839844,
-      "rewards/rejected": -1.42296302318573,
-      "step": 2220
-    },
-    {
-      "epoch": 2.3,
-      "learning_rate": 1.293960244648318e-07,
-      "logits/chosen": -2.62562894821167,
-      "logits/rejected": -2.5972142219543457,
-      "logps/chosen": -373.9268798828125,
-      "logps/rejected": -270.20172119140625,
-      "loss": 0.3315,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.2031903713941574,
-      "rewards/margins": 1.5349807739257812,
-      "rewards/rejected": -1.3317903280258179,
-      "step": 2230
-    },
-    {
-      "epoch": 2.31,
-      "learning_rate": 1.2748470948012232e-07,
-      "logits/chosen": -2.5800623893737793,
-      "logits/rejected": -2.5364508628845215,
-      "logps/chosen": -343.7547302246094,
-      "logps/rejected": -314.71014404296875,
-      "loss": 0.3554,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.10606318712234497,
-      "rewards/margins": 1.4685537815093994,
-      "rewards/rejected": -1.3624905347824097,
-      "step": 2240
-    },
-    {
-      "epoch": 2.32,
-      "learning_rate": 1.2557339449541285e-07,
-      "logits/chosen": -2.6662609577178955,
-      "logits/rejected": -2.6039960384368896,
-      "logps/chosen": -349.7870788574219,
-      "logps/rejected": -287.8039245605469,
-      "loss": 0.3576,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.060789804905653,
-      "rewards/margins": 1.2319402694702148,
-      "rewards/rejected": -1.1711504459381104,
-      "step": 2250
-    },
-    {
-      "epoch": 2.33,
-      "learning_rate": 1.2366207951070336e-07,
-      "logits/chosen": -2.6050796508789062,
-      "logits/rejected": -2.6385793685913086,
-      "logps/chosen": -326.8080139160156,
-      "logps/rejected": -300.26678466796875,
-      "loss": 0.3724,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.1627347767353058,
-      "rewards/margins": 1.3461954593658447,
-      "rewards/rejected": -1.1834605932235718,
-      "step": 2260
-    },
-    {
-      "epoch": 2.34,
-      "learning_rate": 1.217507645259939e-07,
-      "logits/chosen": -2.6019394397735596,
-      "logits/rejected": -2.507327079772949,
-      "logps/chosen": -340.3645935058594,
-      "logps/rejected": -275.542236328125,
-      "loss": 0.3367,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.09731924533843994,
-      "rewards/margins": 1.3692798614501953,
-      "rewards/rejected": -1.2719604969024658,
-      "step": 2270
-    },
-    {
-      "epoch": 2.35,
-      "learning_rate": 1.198394495412844e-07,
-      "logits/chosen": -2.582803726196289,
-      "logits/rejected": -2.577101707458496,
-      "logps/chosen": -358.19354248046875,
-      "logps/rejected": -295.2372131347656,
-      "loss": 0.3408,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.15065257251262665,
-      "rewards/margins": 1.3026378154754639,
-      "rewards/rejected": -1.1519852876663208,
-      "step": 2280
-    },
-    {
-      "epoch": 2.36,
-      "learning_rate": 1.1792813455657493e-07,
-      "logits/chosen": -2.5862011909484863,
-      "logits/rejected": -2.521061420440674,
-      "logps/chosen": -332.89520263671875,
-      "logps/rejected": -267.4942626953125,
-      "loss": 0.3365,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.04419634863734245,
-      "rewards/margins": 1.277284026145935,
-      "rewards/rejected": -1.233087420463562,
-      "step": 2290
-    },
-    {
-      "epoch": 2.37,
-      "learning_rate": 1.1601681957186543e-07,
-      "logits/chosen": -2.5955777168273926,
-      "logits/rejected": -2.5993008613586426,
-      "logps/chosen": -338.1742858886719,
-      "logps/rejected": -275.82666015625,
-      "loss": 0.354,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 0.2074280083179474,
-      "rewards/margins": 1.3635247945785522,
-      "rewards/rejected": -1.1560968160629272,
-      "step": 2300
-    },
-    {
-      "epoch": 2.37,
-      "eval_logits/chosen": -2.707024574279785,
-      "eval_logits/rejected": -2.656226634979248,
-      "eval_logps/chosen": -369.8023986816406,
-      "eval_logps/rejected": -295.6311340332031,
-      "eval_loss": 0.5147502422332764,
-      "eval_rewards/accuracies": 0.7420634627342224,
-      "eval_rewards/chosen": 0.005297229625284672,
-      "eval_rewards/margins": 0.9971935749053955,
-      "eval_rewards/rejected": -0.9918965101242065,
-      "eval_runtime": 93.4767,
-      "eval_samples_per_second": 21.396,
-      "eval_steps_per_second": 0.674,
-      "step": 2300
-    },
-    {
-      "epoch": 2.38,
-      "learning_rate": 1.1410550458715595e-07,
-      "logits/chosen": -2.5364441871643066,
-      "logits/rejected": -2.574249744415283,
-      "logps/chosen": -332.3714904785156,
-      "logps/rejected": -306.153076171875,
-      "loss": 0.3856,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.059642672538757324,
-      "rewards/margins": 1.174212098121643,
-      "rewards/rejected": -1.1145694255828857,
-      "step": 2310
-    },
-    {
-      "epoch": 2.39,
-      "learning_rate": 1.1219418960244648e-07,
-      "logits/chosen": -2.5725884437561035,
-      "logits/rejected": -2.5257275104522705,
-      "logps/chosen": -284.27044677734375,
-      "logps/rejected": -307.5198669433594,
-      "loss": 0.3389,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -0.01629200577735901,
-      "rewards/margins": 1.2315080165863037,
-      "rewards/rejected": -1.2478001117706299,
-      "step": 2320
-    },
-    {
-      "epoch": 2.4,
-      "learning_rate": 1.10282874617737e-07,
-      "logits/chosen": -2.639014720916748,
-      "logits/rejected": -2.545886993408203,
-      "logps/chosen": -304.1851501464844,
-      "logps/rejected": -265.3128356933594,
-      "loss": 0.3175,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.015858350321650505,
-      "rewards/margins": 1.3480350971221924,
-      "rewards/rejected": -1.3321768045425415,
-      "step": 2330
-    },
-    {
-      "epoch": 2.41,
-      "learning_rate": 1.0837155963302752e-07,
-      "logits/chosen": -2.4680936336517334,
-      "logits/rejected": -2.4684300422668457,
-      "logps/chosen": -273.1257019042969,
-      "logps/rejected": -297.2413635253906,
-      "loss": 0.3544,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.0904908999800682,
-      "rewards/margins": 1.409021258354187,
-      "rewards/rejected": -1.318530559539795,
-      "step": 2340
-    },
-    {
-      "epoch": 2.43,
-      "learning_rate": 1.0646024464831804e-07,
-      "logits/chosen": -2.5631706714630127,
-      "logits/rejected": -2.48854398727417,
-      "logps/chosen": -346.63446044921875,
-      "logps/rejected": -282.98809814453125,
-      "loss": 0.3582,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.005453610327094793,
-      "rewards/margins": 1.4462357759475708,
-      "rewards/rejected": -1.4407821893692017,
-      "step": 2350
-    },
-    {
-      "epoch": 2.44,
-      "learning_rate": 1.0454892966360856e-07,
-      "logits/chosen": -2.535862922668457,
-      "logits/rejected": -2.526308536529541,
-      "logps/chosen": -324.5105285644531,
-      "logps/rejected": -272.35546875,
-      "loss": 0.3577,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.08594958484172821,
-      "rewards/margins": 1.5383434295654297,
-      "rewards/rejected": -1.4523937702178955,
-      "step": 2360
-    },
-    {
-      "epoch": 2.45,
-      "learning_rate": 1.0263761467889908e-07,
-      "logits/chosen": -2.477186679840088,
-      "logits/rejected": -2.449218273162842,
-      "logps/chosen": -333.4910888671875,
-      "logps/rejected": -292.4876403808594,
-      "loss": 0.3588,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.16426347196102142,
-      "rewards/margins": 1.3494445085525513,
-      "rewards/rejected": -1.185180902481079,
-      "step": 2370
-    },
-    {
-      "epoch": 2.46,
-      "learning_rate": 1.007262996941896e-07,
-      "logits/chosen": -2.559156894683838,
-      "logits/rejected": -2.4658923149108887,
-      "logps/chosen": -340.9134216308594,
-      "logps/rejected": -260.46282958984375,
-      "loss": 0.3482,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -0.06518319994211197,
-      "rewards/margins": 1.252594232559204,
-      "rewards/rejected": -1.317777395248413,
-      "step": 2380
-    },
-    {
-      "epoch": 2.47,
-      "learning_rate": 9.881498470948011e-08,
-      "logits/chosen": -2.5094966888427734,
-      "logits/rejected": -2.458124876022339,
-      "logps/chosen": -348.07220458984375,
-      "logps/rejected": -263.6317443847656,
-      "loss": 0.3545,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.03768324851989746,
-      "rewards/margins": 1.507036805152893,
-      "rewards/rejected": -1.4693536758422852,
-      "step": 2390
-    },
-    {
-      "epoch": 2.48,
-      "learning_rate": 9.690366972477065e-08,
-      "logits/chosen": -2.616698741912842,
-      "logits/rejected": -2.6090917587280273,
-      "logps/chosen": -346.8231506347656,
-      "logps/rejected": -270.77679443359375,
-      "loss": 0.3539,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.07578043639659882,
-      "rewards/margins": 1.1882234811782837,
-      "rewards/rejected": -1.1124428510665894,
-      "step": 2400
-    },
-    {
-      "epoch": 2.48,
-      "eval_logits/chosen": -2.707021951675415,
-      "eval_logits/rejected": -2.6557092666625977,
-      "eval_logps/chosen": -369.9039001464844,
-      "eval_logps/rejected": -295.6993713378906,
-      "eval_loss": 0.5143767595291138,
-      "eval_rewards/accuracies": 0.738095223903656,
-      "eval_rewards/chosen": -0.004853170830756426,
-      "eval_rewards/margins": 0.9938662052154541,
-      "eval_rewards/rejected": -0.9987194538116455,
-      "eval_runtime": 93.6622,
-      "eval_samples_per_second": 21.353,
-      "eval_steps_per_second": 0.673,
-      "step": 2400
-    },
-    {
-      "epoch": 2.49,
-      "learning_rate": 9.499235474006116e-08,
-      "logits/chosen": -2.618323802947998,
-      "logits/rejected": -2.5537703037261963,
-      "logps/chosen": -379.9132080078125,
-      "logps/rejected": -270.57196044921875,
-      "loss": 0.3367,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.21965615451335907,
-      "rewards/margins": 1.6616443395614624,
-      "rewards/rejected": -1.441988229751587,
-      "step": 2410
-    },
-    {
-      "epoch": 2.5,
-      "learning_rate": 9.308103975535168e-08,
-      "logits/chosen": -2.5823190212249756,
-      "logits/rejected": -2.561429262161255,
-      "logps/chosen": -348.19024658203125,
-      "logps/rejected": -310.19281005859375,
-      "loss": 0.3793,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.07228744775056839,
-      "rewards/margins": 1.1632258892059326,
-      "rewards/rejected": -1.0909385681152344,
-      "step": 2420
-    },
-    {
-      "epoch": 2.51,
-      "learning_rate": 9.116972477064219e-08,
-      "logits/chosen": -2.554273843765259,
-      "logits/rejected": -2.5581376552581787,
-      "logps/chosen": -361.60723876953125,
-      "logps/rejected": -329.79510498046875,
-      "loss": 0.3745,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.07210446894168854,
-      "rewards/margins": 1.3393795490264893,
-      "rewards/rejected": -1.267275094985962,
-      "step": 2430
-    },
-    {
-      "epoch": 2.52,
-      "learning_rate": 8.925840978593272e-08,
-      "logits/chosen": -2.616187572479248,
-      "logits/rejected": -2.5371768474578857,
-      "logps/chosen": -270.7501525878906,
-      "logps/rejected": -303.85870361328125,
-      "loss": 0.3541,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 0.1579769253730774,
-      "rewards/margins": 1.3280938863754272,
-      "rewards/rejected": -1.1701170206069946,
-      "step": 2440
-    },
-    {
-      "epoch": 2.53,
-      "learning_rate": 8.734709480122324e-08,
-      "logits/chosen": -2.507927656173706,
-      "logits/rejected": -2.532486915588379,
-      "logps/chosen": -318.96746826171875,
-      "logps/rejected": -295.8729248046875,
-      "loss": 0.3571,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -0.030292898416519165,
-      "rewards/margins": 1.09835684299469,
-      "rewards/rejected": -1.1286494731903076,
-      "step": 2450
-    },
-    {
-      "epoch": 2.54,
-      "learning_rate": 8.543577981651376e-08,
-      "logits/chosen": -2.575026273727417,
-      "logits/rejected": -2.5608019828796387,
-      "logps/chosen": -447.73883056640625,
-      "logps/rejected": -321.723876953125,
-      "loss": 0.356,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.23453247547149658,
-      "rewards/margins": 1.5171207189559937,
-      "rewards/rejected": -1.2825883626937866,
-      "step": 2460
-    },
-    {
-      "epoch": 2.55,
-      "learning_rate": 8.352446483180428e-08,
-      "logits/chosen": -2.6058762073516846,
-      "logits/rejected": -2.586782932281494,
-      "logps/chosen": -354.8528137207031,
-      "logps/rejected": -262.26611328125,
-      "loss": 0.3642,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.14801561832427979,
-      "rewards/margins": 1.3185254335403442,
-      "rewards/rejected": -1.170509696006775,
-      "step": 2470
-    },
-    {
-      "epoch": 2.56,
-      "learning_rate": 8.161314984709481e-08,
-      "logits/chosen": -2.6247718334198,
-      "logits/rejected": -2.605537176132202,
-      "logps/chosen": -337.4981994628906,
-      "logps/rejected": -287.82305908203125,
-      "loss": 0.3496,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.01781139336526394,
-      "rewards/margins": 1.2502973079681396,
-      "rewards/rejected": -1.2324861288070679,
-      "step": 2480
-    },
-    {
-      "epoch": 2.57,
-      "learning_rate": 7.970183486238531e-08,
-      "logits/chosen": -2.5559067726135254,
-      "logits/rejected": -2.536808729171753,
-      "logps/chosen": -323.12469482421875,
-      "logps/rejected": -272.9129943847656,
-      "loss": 0.3277,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.2089882791042328,
-      "rewards/margins": 1.4187358617782593,
-      "rewards/rejected": -1.209747552871704,
-      "step": 2490
-    },
-    {
-      "epoch": 2.58,
-      "learning_rate": 7.779051987767583e-08,
-      "logits/chosen": -2.652679204940796,
-      "logits/rejected": -2.620300769805908,
-      "logps/chosen": -362.5030822753906,
-      "logps/rejected": -288.15765380859375,
-      "loss": 0.3374,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.24906206130981445,
-      "rewards/margins": 1.4693329334259033,
-      "rewards/rejected": -1.2202708721160889,
-      "step": 2500
-    },
-    {
-      "epoch": 2.58,
-      "eval_logits/chosen": -2.7127866744995117,
-      "eval_logits/rejected": -2.6615724563598633,
-      "eval_logps/chosen": -369.87030029296875,
-      "eval_logps/rejected": -295.8825988769531,
-      "eval_loss": 0.5143249034881592,
-      "eval_rewards/accuracies": 0.75,
-      "eval_rewards/chosen": -0.001490301568992436,
-      "eval_rewards/margins": 1.0155508518218994,
-      "eval_rewards/rejected": -1.0170412063598633,
-      "eval_runtime": 93.6481,
-      "eval_samples_per_second": 21.357,
-      "eval_steps_per_second": 0.673,
-      "step": 2500
-    },
-    {
-      "epoch": 2.59,
-      "learning_rate": 7.587920489296635e-08,
-      "logits/chosen": -2.6370816230773926,
-      "logits/rejected": -2.6355397701263428,
-      "logps/chosen": -350.7646484375,
-      "logps/rejected": -293.7305908203125,
-      "loss": 0.3595,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.11017981916666031,
-      "rewards/margins": 1.314969539642334,
-      "rewards/rejected": -1.2047897577285767,
-      "step": 2510
-    },
-    {
-      "epoch": 2.6,
-      "learning_rate": 7.396788990825688e-08,
-      "logits/chosen": -2.687561273574829,
-      "logits/rejected": -2.678396701812744,
-      "logps/chosen": -331.09088134765625,
-      "logps/rejected": -267.24847412109375,
-      "loss": 0.3673,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.19342190027236938,
-      "rewards/margins": 1.5418986082077026,
-      "rewards/rejected": -1.3484766483306885,
-      "step": 2520
-    },
-    {
-      "epoch": 2.61,
-      "learning_rate": 7.20565749235474e-08,
-      "logits/chosen": -2.540743827819824,
-      "logits/rejected": -2.5151162147521973,
-      "logps/chosen": -314.4068298339844,
-      "logps/rejected": -241.954833984375,
-      "loss": 0.3542,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.1343233287334442,
-      "rewards/margins": 1.3502509593963623,
-      "rewards/rejected": -1.2159278392791748,
-      "step": 2530
-    },
-    {
-      "epoch": 2.62,
-      "learning_rate": 7.014525993883792e-08,
-      "logits/chosen": -2.506711483001709,
-      "logits/rejected": -2.5368690490722656,
-      "logps/chosen": -344.0050354003906,
-      "logps/rejected": -280.0585021972656,
-      "loss": 0.35,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 0.20728519558906555,
-      "rewards/margins": 1.4358912706375122,
-      "rewards/rejected": -1.2286062240600586,
-      "step": 2540
-    },
-    {
-      "epoch": 2.63,
-      "learning_rate": 6.823394495412843e-08,
-      "logits/chosen": -2.5110056400299072,
-      "logits/rejected": -2.5191993713378906,
-      "logps/chosen": -336.6435852050781,
-      "logps/rejected": -278.8410339355469,
-      "loss": 0.3553,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.15368029475212097,
-      "rewards/margins": 1.5688577890396118,
-      "rewards/rejected": -1.415177583694458,
-      "step": 2550
-    },
-    {
-      "epoch": 2.64,
-      "learning_rate": 6.632262996941895e-08,
-      "logits/chosen": -2.514180898666382,
-      "logits/rejected": -2.583381414413452,
-      "logps/chosen": -319.88494873046875,
-      "logps/rejected": -288.6555480957031,
-      "loss": 0.3667,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.12983320653438568,
-      "rewards/margins": 1.418076753616333,
-      "rewards/rejected": -1.2882434129714966,
-      "step": 2560
-    },
-    {
-      "epoch": 2.65,
-      "learning_rate": 6.441131498470948e-08,
-      "logits/chosen": -2.551957607269287,
-      "logits/rejected": -2.5424647331237793,
-      "logps/chosen": -311.9202575683594,
-      "logps/rejected": -276.4605407714844,
-      "loss": 0.3359,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.14209267497062683,
-      "rewards/margins": 1.3686944246292114,
-      "rewards/rejected": -1.2266016006469727,
-      "step": 2570
-    },
-    {
-      "epoch": 2.66,
-      "learning_rate": 6.25e-08,
-      "logits/chosen": -2.5870742797851562,
-      "logits/rejected": -2.5902791023254395,
-      "logps/chosen": -392.72308349609375,
-      "logps/rejected": -302.57098388671875,
-      "loss": 0.3451,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.22271943092346191,
-      "rewards/margins": 1.5374951362609863,
-      "rewards/rejected": -1.3147757053375244,
-      "step": 2580
-    },
-    {
-      "epoch": 2.67,
-      "learning_rate": 6.058868501529052e-08,
-      "logits/chosen": -2.636500597000122,
-      "logits/rejected": -2.6023454666137695,
-      "logps/chosen": -374.46954345703125,
-      "logps/rejected": -298.9087219238281,
-      "loss": 0.3449,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.11469938606023788,
-      "rewards/margins": 1.2926369905471802,
-      "rewards/rejected": -1.177937626838684,
-      "step": 2590
-    },
-    {
-      "epoch": 2.68,
-      "learning_rate": 5.8677370030581035e-08,
-      "logits/chosen": -2.5693795680999756,
-      "logits/rejected": -2.5163960456848145,
-      "logps/chosen": -333.29364013671875,
-      "logps/rejected": -262.6988525390625,
-      "loss": 0.3417,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.1946088969707489,
-      "rewards/margins": 1.3346517086029053,
-      "rewards/rejected": -1.1400429010391235,
-      "step": 2600
-    },
-    {
-      "epoch": 2.68,
-      "eval_logits/chosen": -2.7118382453918457,
-      "eval_logits/rejected": -2.660539150238037,
-      "eval_logps/chosen": -369.8551330566406,
-      "eval_logps/rejected": -295.7532958984375,
-      "eval_loss": 0.5136774778366089,
-      "eval_rewards/accuracies": 0.7341269850730896,
-      "eval_rewards/chosen": 2.2300651835394092e-05,
-      "eval_rewards/margins": 1.0041303634643555,
-      "eval_rewards/rejected": -1.004108190536499,
-      "eval_runtime": 93.7402,
-      "eval_samples_per_second": 21.336,
-      "eval_steps_per_second": 0.672,
-      "step": 2600
-    },
-    {
-      "epoch": 2.69,
-      "learning_rate": 5.6766055045871554e-08,
-      "logits/chosen": -2.571103572845459,
-      "logits/rejected": -2.6057116985321045,
-      "logps/chosen": -340.5564880371094,
-      "logps/rejected": -302.25445556640625,
-      "loss": 0.3308,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.1304255723953247,
-      "rewards/margins": 1.32243013381958,
-      "rewards/rejected": -1.192004680633545,
-      "step": 2610
-    },
-    {
-      "epoch": 2.7,
-      "learning_rate": 5.485474006116208e-08,
-      "logits/chosen": -2.660834550857544,
-      "logits/rejected": -2.553541660308838,
-      "logps/chosen": -352.4358825683594,
-      "logps/rejected": -271.14251708984375,
-      "loss": 0.3525,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.19767490029335022,
-      "rewards/margins": 1.4519338607788086,
-      "rewards/rejected": -1.2542589902877808,
-      "step": 2620
-    },
-    {
-      "epoch": 2.71,
-      "learning_rate": 5.294342507645259e-08,
-      "logits/chosen": -2.588761806488037,
-      "logits/rejected": -2.5700109004974365,
-      "logps/chosen": -398.5816955566406,
-      "logps/rejected": -275.89947509765625,
-      "loss": 0.3466,
-      "rewards/accuracies": 0.925000011920929,
-      "rewards/chosen": 0.3069090247154236,
-      "rewards/margins": 1.7067188024520874,
-      "rewards/rejected": -1.3998098373413086,
-      "step": 2630
-    },
-    {
-      "epoch": 2.72,
-      "learning_rate": 5.1032110091743117e-08,
-      "logits/chosen": -2.434288263320923,
-      "logits/rejected": -2.549377202987671,
-      "logps/chosen": -280.3855895996094,
-      "logps/rejected": -268.7284240722656,
-      "loss": 0.3512,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -0.07931343466043472,
-      "rewards/margins": 1.2083343267440796,
-      "rewards/rejected": -1.2876479625701904,
-      "step": 2640
-    },
-    {
-      "epoch": 2.73,
-      "learning_rate": 4.9120795107033635e-08,
-      "logits/chosen": -2.586907148361206,
-      "logits/rejected": -2.5731730461120605,
-      "logps/chosen": -355.03466796875,
-      "logps/rejected": -296.54437255859375,
-      "loss": 0.3381,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.157174751162529,
-      "rewards/margins": 1.4374758005142212,
-      "rewards/rejected": -1.2803009748458862,
-      "step": 2650
-    },
-    {
-      "epoch": 2.75,
-      "learning_rate": 4.7209480122324154e-08,
-      "logits/chosen": -2.6315078735351562,
-      "logits/rejected": -2.595000743865967,
-      "logps/chosen": -337.0852966308594,
-      "logps/rejected": -308.55670166015625,
-      "loss": 0.3403,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.06230630353093147,
-      "rewards/margins": 1.4226971864700317,
-      "rewards/rejected": -1.3603907823562622,
-      "step": 2660
-    },
-    {
-      "epoch": 2.76,
-      "learning_rate": 4.529816513761467e-08,
-      "logits/chosen": -2.5866010189056396,
-      "logits/rejected": -2.6100072860717773,
-      "logps/chosen": -385.6864013671875,
-      "logps/rejected": -349.44439697265625,
-      "loss": 0.346,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.2403152734041214,
-      "rewards/margins": 1.5250906944274902,
-      "rewards/rejected": -1.2847753763198853,
-      "step": 2670
-    },
-    {
-      "epoch": 2.77,
-      "learning_rate": 4.33868501529052e-08,
-      "logits/chosen": -2.6556694507598877,
-      "logits/rejected": -2.5988481044769287,
-      "logps/chosen": -300.122802734375,
-      "logps/rejected": -253.27743530273438,
-      "loss": 0.3544,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.014878131449222565,
-      "rewards/margins": 1.4045097827911377,
-      "rewards/rejected": -1.3896316289901733,
-      "step": 2680
-    },
-    {
-      "epoch": 2.78,
-      "learning_rate": 4.147553516819572e-08,
-      "logits/chosen": -2.528102159500122,
-      "logits/rejected": -2.514404773712158,
-      "logps/chosen": -293.94830322265625,
-      "logps/rejected": -272.3764343261719,
-      "loss": 0.357,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.14863362908363342,
-      "rewards/margins": 1.587708830833435,
-      "rewards/rejected": -1.439075231552124,
-      "step": 2690
-    },
-    {
-      "epoch": 2.79,
-      "learning_rate": 3.9564220183486236e-08,
-      "logits/chosen": -2.5406494140625,
-      "logits/rejected": -2.4887073040008545,
-      "logps/chosen": -334.87615966796875,
-      "logps/rejected": -244.68984985351562,
-      "loss": 0.3312,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.24883659183979034,
-      "rewards/margins": 1.5429491996765137,
-      "rewards/rejected": -1.2941125631332397,
-      "step": 2700
-    },
-    {
-      "epoch": 2.79,
-      "eval_logits/chosen": -2.7070810794830322,
-      "eval_logits/rejected": -2.6563408374786377,
-      "eval_logps/chosen": -370.0519104003906,
-      "eval_logps/rejected": -295.9976501464844,
-      "eval_loss": 0.5139505863189697,
-      "eval_rewards/accuracies": 0.7301587462425232,
-      "eval_rewards/chosen": -0.019654909148812294,
-      "eval_rewards/margins": 1.0088927745819092,
-      "eval_rewards/rejected": -1.0285476446151733,
-      "eval_runtime": 93.6172,
-      "eval_samples_per_second": 21.364,
-      "eval_steps_per_second": 0.673,
-      "step": 2700
-    },
-    {
-      "epoch": 2.8,
-      "learning_rate": 3.7652905198776755e-08,
-      "logits/chosen": -2.6289658546447754,
-      "logits/rejected": -2.5545971393585205,
-      "logps/chosen": -374.7056579589844,
-      "logps/rejected": -280.23291015625,
-      "loss": 0.3592,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 0.028684472665190697,
-      "rewards/margins": 1.139135718345642,
-      "rewards/rejected": -1.1104512214660645,
-      "step": 2710
-    },
-    {
-      "epoch": 2.81,
-      "learning_rate": 3.574159021406728e-08,
-      "logits/chosen": -2.576387882232666,
-      "logits/rejected": -2.578197956085205,
-      "logps/chosen": -348.49371337890625,
-      "logps/rejected": -368.74249267578125,
-      "loss": 0.3494,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -0.12000823020935059,
-      "rewards/margins": 1.18352210521698,
-      "rewards/rejected": -1.3035303354263306,
-      "step": 2720
-    },
-    {
-      "epoch": 2.82,
-      "learning_rate": 3.383027522935779e-08,
-      "logits/chosen": -2.6115736961364746,
-      "logits/rejected": -2.5528273582458496,
-      "logps/chosen": -349.59222412109375,
-      "logps/rejected": -296.5248107910156,
-      "loss": 0.3612,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.1196526288986206,
-      "rewards/margins": 1.4233744144439697,
-      "rewards/rejected": -1.3037217855453491,
-      "step": 2730
-    },
-    {
-      "epoch": 2.83,
-      "learning_rate": 3.191896024464832e-08,
-      "logits/chosen": -2.579690456390381,
-      "logits/rejected": -2.5436129570007324,
-      "logps/chosen": -358.9446105957031,
-      "logps/rejected": -281.4949645996094,
-      "loss": 0.3526,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.12253328412771225,
-      "rewards/margins": 1.364234209060669,
-      "rewards/rejected": -1.2417008876800537,
-      "step": 2740
-    },
-    {
-      "epoch": 2.84,
-      "learning_rate": 3.0007645259938836e-08,
-      "logits/chosen": -2.5429506301879883,
-      "logits/rejected": -2.530900478363037,
-      "logps/chosen": -280.2276916503906,
-      "logps/rejected": -265.14361572265625,
-      "loss": 0.3422,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.0878460705280304,
-      "rewards/margins": 1.5060606002807617,
-      "rewards/rejected": -1.4182145595550537,
-      "step": 2750
-    },
-    {
-      "epoch": 2.85,
-      "learning_rate": 2.809633027522936e-08,
-      "logits/chosen": -2.5236594676971436,
-      "logits/rejected": -2.5627801418304443,
-      "logps/chosen": -307.91845703125,
-      "logps/rejected": -264.10601806640625,
-      "loss": 0.353,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -0.025409594178199768,
-      "rewards/margins": 1.2293641567230225,
-      "rewards/rejected": -1.2547738552093506,
-      "step": 2760
-    },
-    {
-      "epoch": 2.86,
-      "learning_rate": 2.6185015290519877e-08,
-      "logits/chosen": -2.4620065689086914,
-      "logits/rejected": -2.5668463706970215,
-      "logps/chosen": -343.9793701171875,
-      "logps/rejected": -317.62261962890625,
-      "loss": 0.3397,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/chosen": 0.43111473321914673,
-      "rewards/margins": 1.7761211395263672,
-      "rewards/rejected": -1.3450063467025757,
-      "step": 2770
-    },
-    {
-      "epoch": 2.87,
-      "learning_rate": 2.4273700305810396e-08,
-      "logits/chosen": -2.6023755073547363,
-      "logits/rejected": -2.5225300788879395,
-      "logps/chosen": -391.25018310546875,
-      "logps/rejected": -317.033203125,
-      "loss": 0.3525,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.30640777945518494,
-      "rewards/margins": 1.6793296337127686,
-      "rewards/rejected": -1.3729219436645508,
-      "step": 2780
-    },
-    {
-      "epoch": 2.88,
-      "learning_rate": 2.2362385321100918e-08,
-      "logits/chosen": -2.6185410022735596,
-      "logits/rejected": -2.570345878601074,
-      "logps/chosen": -363.7101135253906,
-      "logps/rejected": -276.9412536621094,
-      "loss": 0.3368,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.16394543647766113,
-      "rewards/margins": 1.499637246131897,
-      "rewards/rejected": -1.3356918096542358,
-      "step": 2790
-    },
-    {
-      "epoch": 2.89,
-      "learning_rate": 2.0451070336391437e-08,
-      "logits/chosen": -2.499922513961792,
-      "logits/rejected": -2.4994664192199707,
-      "logps/chosen": -341.2127990722656,
-      "logps/rejected": -247.0023956298828,
-      "loss": 0.3643,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.09523084759712219,
-      "rewards/margins": 1.2245643138885498,
-      "rewards/rejected": -1.129333734512329,
-      "step": 2800
-    },
-    {
-      "epoch": 2.89,
-      "eval_logits/chosen": -2.706326723098755,
-      "eval_logits/rejected": -2.655164957046509,
-      "eval_logps/chosen": -370.08856201171875,
-      "eval_logps/rejected": -295.9974060058594,
-      "eval_loss": 0.5145964622497559,
-      "eval_rewards/accuracies": 0.7420634627342224,
-      "eval_rewards/chosen": -0.02332073263823986,
-      "eval_rewards/margins": 1.0052008628845215,
-      "eval_rewards/rejected": -1.0285216569900513,
-      "eval_runtime": 93.8959,
-      "eval_samples_per_second": 21.3,
-      "eval_steps_per_second": 0.671,
-      "step": 2800
-    },
-    {
-      "epoch": 2.9,
-      "learning_rate": 1.8539755351681956e-08,
-      "logits/chosen": -2.5391743183135986,
-      "logits/rejected": -2.5156824588775635,
-      "logps/chosen": -338.16632080078125,
-      "logps/rejected": -303.3216552734375,
-      "loss": 0.3278,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -0.011912358924746513,
-      "rewards/margins": 1.2774769067764282,
-      "rewards/rejected": -1.2893892526626587,
-      "step": 2810
-    },
-    {
-      "epoch": 2.91,
-      "learning_rate": 1.6628440366972478e-08,
-      "logits/chosen": -2.6104888916015625,
-      "logits/rejected": -2.520526885986328,
-      "logps/chosen": -363.6390380859375,
-      "logps/rejected": -289.16900634765625,
-      "loss": 0.3387,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/chosen": 0.2589784264564514,
-      "rewards/margins": 1.5904781818389893,
-      "rewards/rejected": -1.3314998149871826,
-      "step": 2820
-    },
-    {
-      "epoch": 2.92,
-      "learning_rate": 1.4717125382262997e-08,
-      "logits/chosen": -2.596144914627075,
-      "logits/rejected": -2.5945043563842773,
-      "logps/chosen": -344.29937744140625,
-      "logps/rejected": -305.58819580078125,
-      "loss": 0.3331,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.24774548411369324,
-      "rewards/margins": 1.5245163440704346,
-      "rewards/rejected": -1.276770830154419,
-      "step": 2830
-    },
-    {
-      "epoch": 2.93,
-      "learning_rate": 1.2805810397553517e-08,
-      "logits/chosen": -2.595520496368408,
-      "logits/rejected": -2.5853350162506104,
-      "logps/chosen": -347.48846435546875,
-      "logps/rejected": -286.7652282714844,
-      "loss": 0.3707,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 0.09414155781269073,
-      "rewards/margins": 1.1994467973709106,
-      "rewards/rejected": -1.1053051948547363,
-      "step": 2840
-    },
-    {
-      "epoch": 2.94,
-      "learning_rate": 1.0894495412844038e-08,
-      "logits/chosen": -2.53163480758667,
-      "logits/rejected": -2.5577445030212402,
-      "logps/chosen": -323.544677734375,
-      "logps/rejected": -286.0074462890625,
-      "loss": 0.3423,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.10755489021539688,
-      "rewards/margins": 1.2161157131195068,
-      "rewards/rejected": -1.1085608005523682,
-      "step": 2850
-    },
-    {
-      "epoch": 2.95,
-      "learning_rate": 8.983180428134555e-09,
-      "logits/chosen": -2.5790271759033203,
-      "logits/rejected": -2.506789207458496,
-      "logps/chosen": -333.0215148925781,
-      "logps/rejected": -236.8868408203125,
-      "loss": 0.3549,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 0.001955956220626831,
-      "rewards/margins": 1.2684563398361206,
-      "rewards/rejected": -1.2665002346038818,
-      "step": 2860
-    },
-    {
-      "epoch": 2.96,
-      "learning_rate": 7.071865443425076e-09,
-      "logits/chosen": -2.5556507110595703,
-      "logits/rejected": -2.4685635566711426,
-      "logps/chosen": -344.0811767578125,
-      "logps/rejected": -286.39019775390625,
-      "loss": 0.3413,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.07706295698881149,
-      "rewards/margins": 1.4479432106018066,
-      "rewards/rejected": -1.370880365371704,
-      "step": 2870
-    },
-    {
-      "epoch": 2.97,
-      "learning_rate": 5.1605504587155965e-09,
-      "logits/chosen": -2.5983853340148926,
-      "logits/rejected": -2.6669363975524902,
-      "logps/chosen": -354.70013427734375,
-      "logps/rejected": -296.3901672363281,
-      "loss": 0.3418,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 0.048993874341249466,
-      "rewards/margins": 1.3146908283233643,
-      "rewards/rejected": -1.2656971216201782,
-      "step": 2880
-    },
-    {
-      "epoch": 2.98,
-      "learning_rate": 3.249235474006116e-09,
-      "logits/chosen": -2.588571071624756,
-      "logits/rejected": -2.5282468795776367,
-      "logps/chosen": -345.6400146484375,
-      "logps/rejected": -299.24481201171875,
-      "loss": 0.3617,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 0.17157995700836182,
-      "rewards/margins": 1.4679298400878906,
-      "rewards/rejected": -1.2963498830795288,
-      "step": 2890
-    },
-    {
-      "epoch": 2.99,
-      "learning_rate": 1.3379204892966359e-09,
-      "logits/chosen": -2.513842821121216,
-      "logits/rejected": -2.489457368850708,
-      "logps/chosen": -330.482177734375,
-      "logps/rejected": -260.58203125,
-      "loss": 0.3322,
-      "rewards/accuracies": 0.9375,
-      "rewards/chosen": 0.15938715636730194,
-      "rewards/margins": 1.6171241998672485,
-      "rewards/rejected": -1.4577369689941406,
-      "step": 2900
-    },
-    {
-      "epoch": 2.99,
-      "eval_logits/chosen": -2.707925319671631,
-      "eval_logits/rejected": -2.657289981842041,
-      "eval_logps/chosen": -370.14801025390625,
-      "eval_logps/rejected": -296.0495910644531,
-      "eval_loss": 0.5141671299934387,
-      "eval_rewards/accuracies": 0.7301587462425232,
-      "eval_rewards/chosen": -0.029261818155646324,
-      "eval_rewards/margins": 1.0044795274734497,
-      "eval_rewards/rejected": -1.0337414741516113,
-      "eval_runtime": 93.5443,
-      "eval_samples_per_second": 21.38,
-      "eval_steps_per_second": 0.673,
-      "step": 2900
-    },
-    {
-      "epoch": 3.0,
-      "step": 2907,
-      "total_flos": 0.0,
-      "train_loss": 0.45207962556766157,
-      "train_runtime": 21062.2956,
-      "train_samples_per_second": 8.826,
-      "train_steps_per_second": 0.138
-    }
-  ],
-  "logging_steps": 10,
-  "max_steps": 2907,
-  "num_train_epochs": 3,
-  "save_steps": 500,
-  "total_flos": 0.0,
-  "trial_name": null,
-  "trial_params": null
-}