{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.998691442030882,
  "eval_steps": 500,
  "global_step": 477,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.010468463752944255,
      "grad_norm": 3.843380266792351,
      "learning_rate": 5.208333333333333e-08,
      "logits/chosen": -2.8176448345184326,
      "logits/rejected": -2.775940418243408,
      "logps/chosen": -1.0736682415008545,
      "logps/rejected": -1.1332499980926514,
      "loss": -0.0244,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -1.0736682415008545,
      "rewards/margins": 0.05958189815282822,
      "rewards/rejected": -1.1332499980926514,
      "step": 5
    },
    {
      "epoch": 0.02093692750588851,
      "grad_norm": 4.13922138458207,
      "learning_rate": 1.0416666666666667e-07,
      "logits/chosen": -2.9982171058654785,
      "logits/rejected": -2.952279806137085,
      "logps/chosen": -1.052896499633789,
      "logps/rejected": -1.194782018661499,
      "loss": -0.0274,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.052896499633789,
      "rewards/margins": 0.1418854296207428,
      "rewards/rejected": -1.194782018661499,
      "step": 10
    },
    {
      "epoch": 0.031405391258832765,
      "grad_norm": 4.388036242315528,
      "learning_rate": 1.5624999999999999e-07,
      "logits/chosen": -2.905001401901245,
      "logits/rejected": -2.881582021713257,
      "logps/chosen": -1.0749495029449463,
      "logps/rejected": -1.1714584827423096,
      "loss": -0.0311,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -1.0749495029449463,
      "rewards/margins": 0.09650889784097672,
      "rewards/rejected": -1.1714584827423096,
      "step": 15
    },
    {
      "epoch": 0.04187385501177702,
      "grad_norm": 4.170757799961158,
      "learning_rate": 2.0833333333333333e-07,
      "logits/chosen": -2.9079184532165527,
      "logits/rejected": -2.850598096847534,
      "logps/chosen": -0.9996750950813293,
      "logps/rejected": -1.0603716373443604,
      "loss": -0.0326,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.9996750950813293,
      "rewards/margins": 0.060696445405483246,
      "rewards/rejected": -1.0603716373443604,
      "step": 20
    },
    {
      "epoch": 0.05234231876472128,
      "grad_norm": 4.2140301502962565,
      "learning_rate": 2.604166666666667e-07,
      "logits/chosen": -2.9368786811828613,
      "logits/rejected": -2.880016565322876,
      "logps/chosen": -0.8968712687492371,
      "logps/rejected": -1.018384337425232,
      "loss": -0.0413,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.8968712687492371,
      "rewards/margins": 0.12151306867599487,
      "rewards/rejected": -1.018384337425232,
      "step": 25
    },
    {
      "epoch": 0.06281078251766553,
      "grad_norm": 5.648380861586244,
      "learning_rate": 3.1249999999999997e-07,
      "logits/chosen": -2.9270145893096924,
      "logits/rejected": -2.897721767425537,
      "logps/chosen": -0.8545770645141602,
      "logps/rejected": -0.9005380868911743,
      "loss": -0.0369,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.8545770645141602,
      "rewards/margins": 0.04596105217933655,
      "rewards/rejected": -0.9005380868911743,
      "step": 30
    },
    {
      "epoch": 0.07327924627060979,
      "grad_norm": 5.194083890339375,
      "learning_rate": 3.645833333333333e-07,
      "logits/chosen": -3.003455638885498,
      "logits/rejected": -2.9747724533081055,
      "logps/chosen": -0.8473809957504272,
      "logps/rejected": -0.9919956922531128,
      "loss": -0.0475,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.8473809957504272,
      "rewards/margins": 0.14461484551429749,
      "rewards/rejected": -0.9919956922531128,
      "step": 35
    },
    {
      "epoch": 0.08374771002355404,
      "grad_norm": 4.598851505341545,
      "learning_rate": 4.1666666666666667e-07,
      "logits/chosen": -2.9274966716766357,
      "logits/rejected": -2.9125096797943115,
      "logps/chosen": -0.875938892364502,
      "logps/rejected": -1.0193151235580444,
      "loss": -0.0484,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.875938892364502,
      "rewards/margins": 0.1433762013912201,
      "rewards/rejected": -1.0193151235580444,
      "step": 40
    },
    {
      "epoch": 0.0942161737764983,
      "grad_norm": 4.789508927144662,
      "learning_rate": 4.6874999999999996e-07,
      "logits/chosen": -2.9352455139160156,
      "logits/rejected": -2.911196231842041,
      "logps/chosen": -0.8146079182624817,
      "logps/rejected": -0.9217915534973145,
      "loss": -0.0617,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.8146079182624817,
      "rewards/margins": 0.10718371719121933,
      "rewards/rejected": -0.9217915534973145,
      "step": 45
    },
    {
      "epoch": 0.10468463752944256,
      "grad_norm": 5.456564382160052,
      "learning_rate": 4.999731868769026e-07,
      "logits/chosen": -2.9189658164978027,
      "logits/rejected": -2.9572436809539795,
      "logps/chosen": -0.8167168498039246,
      "logps/rejected": -1.0531059503555298,
      "loss": -0.0526,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.8167168498039246,
      "rewards/margins": 0.23638899624347687,
      "rewards/rejected": -1.0531059503555298,
      "step": 50
    },
    {
      "epoch": 0.11515310128238682,
      "grad_norm": 7.50323556887671,
      "learning_rate": 4.996716052911017e-07,
      "logits/chosen": -2.9651036262512207,
      "logits/rejected": -2.993089199066162,
      "logps/chosen": -0.9721449017524719,
      "logps/rejected": -1.0521472692489624,
      "loss": -0.0482,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.9721449017524719,
      "rewards/margins": 0.08000238239765167,
      "rewards/rejected": -1.0521472692489624,
      "step": 55
    },
    {
      "epoch": 0.12562156503533106,
      "grad_norm": 8.485162916006143,
      "learning_rate": 4.990353313429303e-07,
      "logits/chosen": -2.9055917263031006,
      "logits/rejected": -2.9019391536712646,
      "logps/chosen": -0.9157769083976746,
      "logps/rejected": -1.0977026224136353,
      "loss": -0.0682,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.9157769083976746,
      "rewards/margins": 0.18192583322525024,
      "rewards/rejected": -1.0977026224136353,
      "step": 60
    },
    {
      "epoch": 0.1360900287882753,
      "grad_norm": 6.60792180172325,
      "learning_rate": 4.980652179769217e-07,
      "logits/chosen": -2.966867446899414,
      "logits/rejected": -2.991152048110962,
      "logps/chosen": -0.953170895576477,
      "logps/rejected": -1.0807805061340332,
      "loss": -0.0605,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.953170895576477,
      "rewards/margins": 0.12760967016220093,
      "rewards/rejected": -1.0807805061340332,
      "step": 65
    },
    {
      "epoch": 0.14655849254121958,
      "grad_norm": 7.134153653486866,
      "learning_rate": 4.967625656594781e-07,
      "logits/chosen": -2.9338037967681885,
      "logits/rejected": -2.921638011932373,
      "logps/chosen": -0.9601171612739563,
      "logps/rejected": -1.170008659362793,
      "loss": -0.0809,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.9601171612739563,
      "rewards/margins": 0.20989160239696503,
      "rewards/rejected": -1.170008659362793,
      "step": 70
    },
    {
      "epoch": 0.15702695629416383,
      "grad_norm": 8.86078567599782,
      "learning_rate": 4.951291206355559e-07,
      "logits/chosen": -2.9449570178985596,
      "logits/rejected": -2.863473415374756,
      "logps/chosen": -1.019167423248291,
      "logps/rejected": -1.3563578128814697,
      "loss": -0.0632,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.019167423248291,
      "rewards/margins": 0.3371904492378235,
      "rewards/rejected": -1.3563578128814697,
      "step": 75
    },
    {
      "epoch": 0.16749542004710807,
      "grad_norm": 6.323638159923026,
      "learning_rate": 4.93167072587771e-07,
      "logits/chosen": -2.9299821853637695,
      "logits/rejected": -2.91919207572937,
      "logps/chosen": -0.993580162525177,
      "logps/rejected": -1.4255390167236328,
      "loss": -0.0782,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.993580162525177,
      "rewards/margins": 0.43195900321006775,
      "rewards/rejected": -1.4255390167236328,
      "step": 80
    },
    {
      "epoch": 0.17796388380005235,
      "grad_norm": 10.148627109182378,
      "learning_rate": 4.908790517010636e-07,
      "logits/chosen": -2.9560999870300293,
      "logits/rejected": -2.9236154556274414,
      "logps/chosen": -0.9319014549255371,
      "logps/rejected": -1.2665565013885498,
      "loss": -0.0829,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.9319014549255371,
      "rewards/margins": 0.3346549868583679,
      "rewards/rejected": -1.2665565013885498,
      "step": 85
    },
    {
      "epoch": 0.1884323475529966,
      "grad_norm": 6.282748689638336,
      "learning_rate": 4.882681251368548e-07,
      "logits/chosen": -3.0003504753112793,
      "logits/rejected": -2.9371325969696045,
      "logps/chosen": -0.9956916570663452,
      "logps/rejected": -1.5206801891326904,
      "loss": -0.085,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.9956916570663452,
      "rewards/margins": 0.5249885320663452,
      "rewards/rejected": -1.5206801891326904,
      "step": 90
    },
    {
      "epoch": 0.19890081130594087,
      "grad_norm": 6.0015528777597265,
      "learning_rate": 4.853377929214243e-07,
      "logits/chosen": -2.938925266265869,
      "logits/rejected": -2.9378859996795654,
      "logps/chosen": -0.9370375871658325,
      "logps/rejected": -1.2533988952636719,
      "loss": -0.073,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.9370375871658325,
      "rewards/margins": 0.3163612484931946,
      "rewards/rejected": -1.2533988952636719,
      "step": 95
    },
    {
      "epoch": 0.2093692750588851,
      "grad_norm": 9.968256329447078,
      "learning_rate": 4.820919832540181e-07,
      "logits/chosen": -3.1243631839752197,
      "logits/rejected": -3.087338924407959,
      "logps/chosen": -0.9904219508171082,
      "logps/rejected": -1.245534062385559,
      "loss": -0.0777,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.9904219508171082,
      "rewards/margins": 0.2551119923591614,
      "rewards/rejected": -1.245534062385559,
      "step": 100
    },
    {
      "epoch": 0.21983773881182936,
      "grad_norm": 13.342769400160067,
      "learning_rate": 4.785350472409791e-07,
      "logits/chosen": -2.987121105194092,
      "logits/rejected": -3.0028722286224365,
      "logps/chosen": -1.1532871723175049,
      "logps/rejected": -1.6190894842147827,
      "loss": -0.1063,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1532871723175049,
      "rewards/margins": 0.4658023715019226,
      "rewards/rejected": -1.6190894842147827,
      "step": 105
    },
    {
      "epoch": 0.23030620256477363,
      "grad_norm": 11.26110556096259,
      "learning_rate": 4.7467175306295647e-07,
      "logits/chosen": -2.966667652130127,
      "logits/rejected": -3.0194077491760254,
      "logps/chosen": -1.0292152166366577,
      "logps/rejected": -1.59182608127594,
      "loss": -0.1066,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.0292152166366577,
      "rewards/margins": 0.562610924243927,
      "rewards/rejected": -1.59182608127594,
      "step": 110
    },
    {
      "epoch": 0.24077466631771788,
      "grad_norm": 8.176863863279703,
      "learning_rate": 4.70507279583015e-07,
      "logits/chosen": -3.0338950157165527,
      "logits/rejected": -3.0541911125183105,
      "logps/chosen": -1.1478749513626099,
      "logps/rejected": -1.4170184135437012,
      "loss": -0.0877,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.1478749513626099,
      "rewards/margins": 0.2691434919834137,
      "rewards/rejected": -1.4170184135437012,
      "step": 115
    },
    {
      "epoch": 0.2512431300706621,
      "grad_norm": 8.289689216777566,
      "learning_rate": 4.6604720940421207e-07,
      "logits/chosen": -3.0846564769744873,
      "logits/rejected": -3.0871291160583496,
      "logps/chosen": -1.1712749004364014,
      "logps/rejected": -1.6515331268310547,
      "loss": -0.0702,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.1712749004364014,
      "rewards/margins": 0.48025816679000854,
      "rewards/rejected": -1.6515331268310547,
      "step": 120
    },
    {
      "epoch": 0.26171159382360637,
      "grad_norm": 8.632430270339013,
      "learning_rate": 4.612975213859487e-07,
      "logits/chosen": -3.0736565589904785,
      "logits/rejected": -3.0235087871551514,
      "logps/chosen": -1.246620535850525,
      "logps/rejected": -1.603502869606018,
      "loss": -0.0801,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.246620535850525,
      "rewards/margins": 0.3568824827671051,
      "rewards/rejected": -1.603502869606018,
      "step": 125
    },
    {
      "epoch": 0.2721800575765506,
      "grad_norm": 6.951210195379089,
      "learning_rate": 4.5626458262912735e-07,
      "logits/chosen": -3.0484471321105957,
      "logits/rejected": -3.0786657333374023,
      "logps/chosen": -1.2631396055221558,
      "logps/rejected": -1.8509140014648438,
      "loss": -0.1013,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.2631396055221558,
      "rewards/margins": 0.5877745747566223,
      "rewards/rejected": -1.8509140014648438,
      "step": 130
    },
    {
      "epoch": 0.2826485213294949,
      "grad_norm": 6.6831446876498966,
      "learning_rate": 4.5095513994085974e-07,
      "logits/chosen": -2.9543702602386475,
      "logits/rejected": -2.9393744468688965,
      "logps/chosen": -1.0893765687942505,
      "logps/rejected": -1.7171491384506226,
      "loss": -0.0945,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.0893765687942505,
      "rewards/margins": 0.6277726292610168,
      "rewards/rejected": -1.7171491384506226,
      "step": 135
    },
    {
      "epoch": 0.29311698508243916,
      "grad_norm": 11.237918474787836,
      "learning_rate": 4.453763107901675e-07,
      "logits/chosen": -3.0231282711029053,
      "logits/rejected": -3.0941128730773926,
      "logps/chosen": -1.3833149671554565,
      "logps/rejected": -1.7585477828979492,
      "loss": -0.0791,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -1.3833149671554565,
      "rewards/margins": 0.37523263692855835,
      "rewards/rejected": -1.7585477828979492,
      "step": 140
    },
    {
      "epoch": 0.3035854488353834,
      "grad_norm": 11.485846701864144,
      "learning_rate": 4.395355737667985e-07,
      "logits/chosen": -3.0041990280151367,
      "logits/rejected": -3.0510060787200928,
      "logps/chosen": -1.1283174753189087,
      "logps/rejected": -1.8526252508163452,
      "loss": -0.1012,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.1283174753189087,
      "rewards/margins": 0.7243077158927917,
      "rewards/rejected": -1.8526252508163452,
      "step": 145
    },
    {
      "epoch": 0.31405391258832765,
      "grad_norm": 11.357824867739286,
      "learning_rate": 4.3344075855595097e-07,
      "logits/chosen": -3.0792462825775146,
      "logits/rejected": -3.0518386363983154,
      "logps/chosen": -1.247040033340454,
      "logps/rejected": -1.7509548664093018,
      "loss": -0.0889,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.247040033340454,
      "rewards/margins": 0.5039147734642029,
      "rewards/rejected": -1.7509548664093018,
      "step": 150
    },
    {
      "epoch": 0.3245223763412719,
      "grad_norm": 11.335355380541145,
      "learning_rate": 4.271000354423425e-07,
      "logits/chosen": -2.9956650733947754,
      "logits/rejected": -3.0046334266662598,
      "logps/chosen": -1.3198177814483643,
      "logps/rejected": -2.0559017658233643,
      "loss": -0.111,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.3198177814483643,
      "rewards/margins": 0.7360838651657104,
      "rewards/rejected": -2.0559017658233643,
      "step": 155
    },
    {
      "epoch": 0.33499084009421615,
      "grad_norm": 16.693424155067344,
      "learning_rate": 4.2052190435769554e-07,
      "logits/chosen": -2.9203574657440186,
      "logits/rejected": -2.9545798301696777,
      "logps/chosen": -1.1988856792449951,
      "logps/rejected": -1.823240876197815,
      "loss": -0.1107,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.1988856792449951,
      "rewards/margins": 0.6243550181388855,
      "rewards/rejected": -1.823240876197815,
      "step": 160
    },
    {
      "epoch": 0.34545930384716045,
      "grad_norm": 26.190973492572102,
      "learning_rate": 4.137151834863213e-07,
      "logits/chosen": -2.992180347442627,
      "logits/rejected": -2.9608774185180664,
      "logps/chosen": -1.3537108898162842,
      "logps/rejected": -1.587428331375122,
      "loss": -0.114,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -1.3537108898162842,
      "rewards/margins": 0.2337172031402588,
      "rewards/rejected": -1.587428331375122,
      "step": 165
    },
    {
      "epoch": 0.3559277676001047,
      "grad_norm": 11.938074204901223,
      "learning_rate": 4.0668899744407567e-07,
      "logits/chosen": -2.943603992462158,
      "logits/rejected": -2.9042210578918457,
      "logps/chosen": -1.1778740882873535,
      "logps/rejected": -1.7121427059173584,
      "loss": -0.1062,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1778740882873535,
      "rewards/margins": 0.5342684984207153,
      "rewards/rejected": -1.7121427059173584,
      "step": 170
    },
    {
      "epoch": 0.36639623135304894,
      "grad_norm": 9.013768613990127,
      "learning_rate": 3.994527650465352e-07,
      "logits/chosen": -2.767575263977051,
      "logits/rejected": -2.7348756790161133,
      "logps/chosen": -1.2128514051437378,
      "logps/rejected": -2.0194027423858643,
      "loss": -0.1222,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.2128514051437378,
      "rewards/margins": 0.8065516352653503,
      "rewards/rejected": -2.0194027423858643,
      "step": 175
    },
    {
      "epoch": 0.3768646951059932,
      "grad_norm": 20.304059925871332,
      "learning_rate": 3.920161866827889e-07,
      "logits/chosen": -2.6296277046203613,
      "logits/rejected": -2.5583887100219727,
      "logps/chosen": -1.4558690786361694,
      "logps/rejected": -2.660147190093994,
      "loss": -0.104,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.4558690786361694,
      "rewards/margins": 1.2042779922485352,
      "rewards/rejected": -2.660147190093994,
      "step": 180
    },
    {
      "epoch": 0.38733315885893743,
      "grad_norm": 12.56852650611151,
      "learning_rate": 3.8438923131177237e-07,
      "logits/chosen": -2.646308660507202,
      "logits/rejected": -2.5967037677764893,
      "logps/chosen": -1.2808468341827393,
      "logps/rejected": -2.6707491874694824,
      "loss": -0.1264,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.2808468341827393,
      "rewards/margins": 1.3899024724960327,
      "rewards/rejected": -2.6707491874694824,
      "step": 185
    },
    {
      "epoch": 0.39780162261188173,
      "grad_norm": 10.671381825602081,
      "learning_rate": 3.765821230985757e-07,
      "logits/chosen": -2.6582224369049072,
      "logits/rejected": -2.643031358718872,
      "logps/chosen": -1.2568540573120117,
      "logps/rejected": -2.3855767250061035,
      "loss": -0.1248,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.2568540573120117,
      "rewards/margins": 1.1287227869033813,
      "rewards/rejected": -2.3855767250061035,
      "step": 190
    },
    {
      "epoch": 0.408270086364826,
      "grad_norm": 12.305445384315465,
      "learning_rate": 3.6860532770864005e-07,
      "logits/chosen": -2.6588711738586426,
      "logits/rejected": -2.650085926055908,
      "logps/chosen": -1.2373688220977783,
      "logps/rejected": -2.440690755844116,
      "loss": -0.1086,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.2373688220977783,
      "rewards/margins": 1.2033220529556274,
      "rewards/rejected": -2.440690755844116,
      "step": 195
    },
    {
      "epoch": 0.4187385501177702,
      "grad_norm": 114.17793963840649,
      "learning_rate": 3.604695382782159e-07,
      "logits/chosen": -2.5837745666503906,
      "logits/rejected": -2.5190796852111816,
      "logps/chosen": -1.535466194152832,
      "logps/rejected": -2.662421703338623,
      "loss": -0.1231,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.535466194152832,
      "rewards/margins": 1.126955509185791,
      "rewards/rejected": -2.662421703338623,
      "step": 200
    },
    {
      "epoch": 0.42920701387071447,
      "grad_norm": 29.258244612720908,
      "learning_rate": 3.5218566107988867e-07,
      "logits/chosen": -2.4933407306671143,
      "logits/rejected": -2.3973355293273926,
      "logps/chosen": -1.6263624429702759,
      "logps/rejected": -3.2516510486602783,
      "loss": -0.1144,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.6263624429702759,
      "rewards/margins": 1.6252882480621338,
      "rewards/rejected": -3.2516510486602783,
      "step": 205
    },
    {
      "epoch": 0.4396754776236587,
      "grad_norm": 39.526967700856005,
      "learning_rate": 3.4376480090239047e-07,
      "logits/chosen": -2.342941999435425,
      "logits/rejected": -2.3288662433624268,
      "logps/chosen": -2.3925395011901855,
      "logps/rejected": -3.234570264816284,
      "loss": -0.0999,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -2.3925395011901855,
      "rewards/margins": 0.8420308232307434,
      "rewards/rejected": -3.234570264816284,
      "step": 210
    },
    {
      "epoch": 0.45014394137660296,
      "grad_norm": 15.657987448676648,
      "learning_rate": 3.3521824616429284e-07,
      "logits/chosen": -2.3377509117126465,
      "logits/rejected": -2.3475136756896973,
      "logps/chosen": -2.209704875946045,
      "logps/rejected": -3.1227197647094727,
      "loss": -0.1112,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.209704875946045,
      "rewards/margins": 0.9130150079727173,
      "rewards/rejected": -3.1227197647094727,
      "step": 215
    },
    {
      "epoch": 0.46061240512954726,
      "grad_norm": 24.63970678663639,
      "learning_rate": 3.265574537815398e-07,
      "logits/chosen": -2.3190903663635254,
      "logits/rejected": -2.222909927368164,
      "logps/chosen": -1.5355461835861206,
      "logps/rejected": -2.3923590183258057,
      "loss": -0.125,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.5355461835861206,
      "rewards/margins": 0.8568128347396851,
      "rewards/rejected": -2.3923590183258057,
      "step": 220
    },
    {
      "epoch": 0.4710808688824915,
      "grad_norm": 10.677074157142071,
      "learning_rate": 3.1779403380910425e-07,
      "logits/chosen": -2.237248182296753,
      "logits/rejected": -2.051560163497925,
      "logps/chosen": -1.662973165512085,
      "logps/rejected": -2.656738519668579,
      "loss": -0.1184,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.662973165512085,
      "rewards/margins": 0.9937652349472046,
      "rewards/rejected": -2.656738519668579,
      "step": 225
    },
    {
      "epoch": 0.48154933263543576,
      "grad_norm": 63.22829689035937,
      "learning_rate": 3.0893973387735683e-07,
      "logits/chosen": -2.1220011711120605,
      "logits/rejected": -1.9981296062469482,
      "logps/chosen": -1.8179805278778076,
      "logps/rejected": -3.1246554851531982,
      "loss": -0.1233,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.8179805278778076,
      "rewards/margins": 1.3066750764846802,
      "rewards/rejected": -3.1246554851531982,
      "step": 230
    },
    {
      "epoch": 0.49201779638838,
      "grad_norm": 16.030732819551737,
      "learning_rate": 3.000064234440111e-07,
      "logits/chosen": -2.058353900909424,
      "logits/rejected": -1.8866300582885742,
      "logps/chosen": -1.891436219215393,
      "logps/rejected": -2.721259832382202,
      "loss": -0.1073,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.891436219215393,
      "rewards/margins": 0.8298239707946777,
      "rewards/rejected": -2.721259832382202,
      "step": 235
    },
    {
      "epoch": 0.5024862601413242,
      "grad_norm": 13.634356149094991,
      "learning_rate": 2.910060778827554e-07,
      "logits/chosen": -1.8825008869171143,
      "logits/rejected": -1.8133189678192139,
      "logps/chosen": -2.548449993133545,
      "logps/rejected": -3.235548496246338,
      "loss": -0.1088,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.548449993133545,
      "rewards/margins": 0.6870983839035034,
      "rewards/rejected": -3.235548496246338,
      "step": 240
    },
    {
      "epoch": 0.5129547238942685,
      "grad_norm": 29.272649537767588,
      "learning_rate": 2.8195076242990116e-07,
      "logits/chosen": -2.0280842781066895,
      "logits/rejected": -1.8670088052749634,
      "logps/chosen": -1.8938684463500977,
      "logps/rejected": -3.3442654609680176,
      "loss": -0.1234,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.8938684463500977,
      "rewards/margins": 1.4503968954086304,
      "rewards/rejected": -3.3442654609680176,
      "step": 245
    },
    {
      "epoch": 0.5234231876472127,
      "grad_norm": 20.57218560544621,
      "learning_rate": 2.7285261601056697e-07,
      "logits/chosen": -2.137603759765625,
      "logits/rejected": -1.8585504293441772,
      "logps/chosen": -1.5400477647781372,
      "logps/rejected": -3.192748546600342,
      "loss": -0.1016,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.5400477647781372,
      "rewards/margins": 1.6527007818222046,
      "rewards/rejected": -3.192748546600342,
      "step": 250
    },
    {
      "epoch": 0.533891651400157,
      "grad_norm": 32.87314594164391,
      "learning_rate": 2.6372383496608186e-07,
      "logits/chosen": -2.2034177780151367,
      "logits/rejected": -2.07490873336792,
      "logps/chosen": -1.890310525894165,
      "logps/rejected": -2.4908692836761475,
      "loss": -0.1104,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -1.890310525894165,
      "rewards/margins": 0.6005589365959167,
      "rewards/rejected": -2.4908692836761475,
      "step": 255
    },
    {
      "epoch": 0.5443601151531012,
      "grad_norm": 13.702145049518174,
      "learning_rate": 2.5457665670441937e-07,
      "logits/chosen": -1.9207513332366943,
      "logits/rejected": -1.8922897577285767,
      "logps/chosen": -1.7071243524551392,
      "logps/rejected": -2.4078123569488525,
      "loss": -0.1183,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.7071243524551392,
      "rewards/margins": 0.7006877064704895,
      "rewards/rejected": -2.4078123569488525,
      "step": 260
    },
    {
      "epoch": 0.5548285789060455,
      "grad_norm": 38.3020358782431,
      "learning_rate": 2.454233432955807e-07,
      "logits/chosen": -2.05800199508667,
      "logits/rejected": -2.0371463298797607,
      "logps/chosen": -1.9121761322021484,
      "logps/rejected": -2.3142733573913574,
      "loss": -0.1309,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.9121761322021484,
      "rewards/margins": 0.4020973742008209,
      "rewards/rejected": -2.3142733573913574,
      "step": 265
    },
    {
      "epoch": 0.5652970426589898,
      "grad_norm": 24.19175613282827,
      "learning_rate": 2.3627616503391812e-07,
      "logits/chosen": -1.966522455215454,
      "logits/rejected": -2.0543503761291504,
      "logps/chosen": -1.5488941669464111,
      "logps/rejected": -2.404010534286499,
      "loss": -0.1062,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.5488941669464111,
      "rewards/margins": 0.8551164865493774,
      "rewards/rejected": -2.404010534286499,
      "step": 270
    },
    {
      "epoch": 0.575765506411934,
      "grad_norm": 32.544264458959944,
      "learning_rate": 2.2714738398943308e-07,
      "logits/chosen": -2.1266486644744873,
      "logits/rejected": -1.9693682193756104,
      "logps/chosen": -1.345821499824524,
      "logps/rejected": -2.7840895652770996,
      "loss": -0.1459,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.345821499824524,
      "rewards/margins": 1.4382681846618652,
      "rewards/rejected": -2.7840895652770996,
      "step": 275
    },
    {
      "epoch": 0.5862339701648783,
      "grad_norm": 21.822579332484686,
      "learning_rate": 2.1804923757009882e-07,
      "logits/chosen": -2.1691737174987793,
      "logits/rejected": -1.9860219955444336,
      "logps/chosen": -1.2002053260803223,
      "logps/rejected": -2.559629201889038,
      "loss": -0.1202,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.2002053260803223,
      "rewards/margins": 1.359424114227295,
      "rewards/rejected": -2.559629201889038,
      "step": 280
    },
    {
      "epoch": 0.5967024339178225,
      "grad_norm": 35.5262923825776,
      "learning_rate": 2.089939221172446e-07,
      "logits/chosen": -2.1386303901672363,
      "logits/rejected": -2.101440906524658,
      "logps/chosen": -1.337780237197876,
      "logps/rejected": -2.1833016872406006,
      "loss": -0.1175,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.337780237197876,
      "rewards/margins": 0.8455215692520142,
      "rewards/rejected": -2.1833016872406006,
      "step": 285
    },
    {
      "epoch": 0.6071708976707668,
      "grad_norm": 27.60629251004708,
      "learning_rate": 1.9999357655598891e-07,
      "logits/chosen": -2.098567008972168,
      "logits/rejected": -1.9679734706878662,
      "logps/chosen": -1.6509898900985718,
      "logps/rejected": -2.216376781463623,
      "loss": -0.1164,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.6509898900985718,
      "rewards/margins": 0.5653868913650513,
      "rewards/rejected": -2.216376781463623,
      "step": 290
    },
    {
      "epoch": 0.6176393614237111,
      "grad_norm": 29.13174757981831,
      "learning_rate": 1.9106026612264315e-07,
      "logits/chosen": -1.9900572299957275,
      "logits/rejected": -1.8516021966934204,
      "logps/chosen": -1.4829915761947632,
      "logps/rejected": -2.072486400604248,
      "loss": -0.1277,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.4829915761947632,
      "rewards/margins": 0.5894945859909058,
      "rewards/rejected": -2.072486400604248,
      "step": 295
    },
    {
      "epoch": 0.6281078251766553,
      "grad_norm": 18.174081755957985,
      "learning_rate": 1.8220596619089573e-07,
      "logits/chosen": -1.8426250219345093,
      "logits/rejected": -1.7023963928222656,
      "logps/chosen": -1.5617187023162842,
      "logps/rejected": -2.6344046592712402,
      "loss": -0.1179,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.5617187023162842,
      "rewards/margins": 1.072685956954956,
      "rewards/rejected": -2.6344046592712402,
      "step": 300
    },
    {
      "epoch": 0.6385762889295996,
      "grad_norm": 23.872227813445928,
      "learning_rate": 1.7344254621846017e-07,
      "logits/chosen": -1.8805307149887085,
      "logits/rejected": -1.748567819595337,
      "logps/chosen": -1.3314863443374634,
      "logps/rejected": -1.9546124935150146,
      "loss": -0.1335,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.3314863443374634,
      "rewards/margins": 0.623126208782196,
      "rewards/rejected": -1.9546124935150146,
      "step": 305
    },
    {
      "epoch": 0.6490447526825438,
      "grad_norm": 29.431070606486642,
      "learning_rate": 1.647817538357072e-07,
      "logits/chosen": -1.8305540084838867,
      "logits/rejected": -1.5959422588348389,
      "logps/chosen": -1.4600447416305542,
      "logps/rejected": -3.613452196121216,
      "loss": -0.1338,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.4600447416305542,
      "rewards/margins": 2.153407573699951,
      "rewards/rejected": -3.613452196121216,
      "step": 310
    },
    {
      "epoch": 0.6595132164354881,
      "grad_norm": 20.33172315066391,
      "learning_rate": 1.562351990976095e-07,
      "logits/chosen": -1.7097995281219482,
      "logits/rejected": -1.444483995437622,
      "logps/chosen": -1.8937146663665771,
      "logps/rejected": -2.9181923866271973,
      "loss": -0.147,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.8937146663665771,
      "rewards/margins": 1.0244777202606201,
      "rewards/rejected": -2.9181923866271973,
      "step": 315
    },
    {
      "epoch": 0.6699816801884323,
      "grad_norm": 11.69882174941769,
      "learning_rate": 1.478143389201113e-07,
      "logits/chosen": -1.5626455545425415,
      "logits/rejected": -1.408778190612793,
      "logps/chosen": -1.8121535778045654,
      "logps/rejected": -2.9141504764556885,
      "loss": -0.1327,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.8121535778045654,
      "rewards/margins": 1.1019970178604126,
      "rewards/rejected": -2.9141504764556885,
      "step": 320
    },
    {
      "epoch": 0.6804501439413766,
      "grad_norm": 31.4758832690907,
      "learning_rate": 1.3953046172178413e-07,
      "logits/chosen": -1.538895606994629,
      "logits/rejected": -1.3528716564178467,
      "logps/chosen": -1.577697515487671,
      "logps/rejected": -3.2226383686065674,
      "loss": -0.108,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.577697515487671,
      "rewards/margins": 1.6449410915374756,
      "rewards/rejected": -3.2226383686065674,
      "step": 325
    },
    {
      "epoch": 0.6909186076943209,
      "grad_norm": 30.00262748267467,
      "learning_rate": 1.3139467229135998e-07,
      "logits/chosen": -1.633194923400879,
      "logits/rejected": -1.5735431909561157,
      "logps/chosen": -1.4919952154159546,
      "logps/rejected": -3.3074562549591064,
      "loss": -0.1222,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.4919952154159546,
      "rewards/margins": 1.8154609203338623,
      "rewards/rejected": -3.3074562549591064,
      "step": 330
    },
    {
      "epoch": 0.7013870714472651,
      "grad_norm": 148.5006608776355,
      "learning_rate": 1.2341787690142435e-07,
      "logits/chosen": -1.653297781944275,
      "logits/rejected": -1.3402432203292847,
      "logps/chosen": -1.81472647190094,
      "logps/rejected": -3.3855223655700684,
      "loss": -0.118,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.81472647190094,
      "rewards/margins": 1.5707957744598389,
      "rewards/rejected": -3.3855223655700684,
      "step": 335
    },
    {
      "epoch": 0.7118555352002094,
      "grad_norm": 19.087270572643252,
      "learning_rate": 1.1561076868822755e-07,
      "logits/chosen": -1.3242827653884888,
      "logits/rejected": -1.1375129222869873,
      "logps/chosen": -1.8913873434066772,
      "logps/rejected": -3.8381896018981934,
      "loss": -0.1367,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.8913873434066772,
      "rewards/margins": 1.9468021392822266,
      "rewards/rejected": -3.8381896018981934,
      "step": 340
    },
    {
      "epoch": 0.7223239989531536,
      "grad_norm": 12.352384869534044,
      "learning_rate": 1.0798381331721107e-07,
      "logits/chosen": -1.622854471206665,
      "logits/rejected": -1.4534351825714111,
      "logps/chosen": -1.520581603050232,
      "logps/rejected": -2.94417142868042,
      "loss": -0.1259,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.520581603050232,
      "rewards/margins": 1.423590064048767,
      "rewards/rejected": -2.94417142868042,
      "step": 345
    },
    {
      "epoch": 0.7327924627060979,
      "grad_norm": 14.805439320214807,
      "learning_rate": 1.0054723495346482e-07,
      "logits/chosen": -1.6887247562408447,
      "logits/rejected": -1.1422927379608154,
      "logps/chosen": -1.5419939756393433,
      "logps/rejected": -3.1307151317596436,
      "loss": -0.1286,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5419939756393433,
      "rewards/margins": 1.5887210369110107,
      "rewards/rejected": -3.1307151317596436,
      "step": 350
    },
    {
      "epoch": 0.7432609264590422,
      "grad_norm": 20.09481183401406,
      "learning_rate": 9.331100255592436e-08,
      "logits/chosen": -1.481506109237671,
      "logits/rejected": -1.2393898963928223,
      "logps/chosen": -1.6499707698822021,
      "logps/rejected": -2.86677885055542,
      "loss": -0.1247,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.6499707698822021,
      "rewards/margins": 1.2168083190917969,
      "rewards/rejected": -2.86677885055542,
      "step": 355
    },
    {
      "epoch": 0.7537293902119864,
      "grad_norm": 21.363222549077037,
      "learning_rate": 8.628481651367875e-08,
      "logits/chosen": -1.531849980354309,
      "logits/rejected": -1.4370256662368774,
      "logps/chosen": -1.5883554220199585,
      "logps/rejected": -3.119661808013916,
      "loss": -0.1282,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.5883554220199585,
      "rewards/margins": 1.531306505203247,
      "rewards/rejected": -3.119661808013916,
      "step": 360
    },
    {
      "epoch": 0.7641978539649307,
      "grad_norm": 24.973104208608277,
      "learning_rate": 7.947809564230445e-08,
      "logits/chosen": -1.7802162170410156,
      "logits/rejected": -1.3090742826461792,
      "logps/chosen": -1.4277862310409546,
      "logps/rejected": -3.4141273498535156,
      "loss": -0.1417,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.4277862310409546,
      "rewards/margins": 1.986340880393982,
      "rewards/rejected": -3.4141273498535156,
      "step": 365
    },
    {
      "epoch": 0.7746663177178749,
      "grad_norm": 20.21283934356993,
      "learning_rate": 7.289996455765748e-08,
      "logits/chosen": -1.6334741115570068,
      "logits/rejected": -1.4152132272720337,
      "logps/chosen": -1.3029416799545288,
      "logps/rejected": -2.3641180992126465,
      "loss": -0.162,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.3029416799545288,
      "rewards/margins": 1.0611765384674072,
      "rewards/rejected": -2.3641180992126465,
      "step": 370
    },
    {
      "epoch": 0.7851347814708192,
      "grad_norm": 19.15939039084446,
      "learning_rate": 6.655924144404906e-08,
      "logits/chosen": -1.5635571479797363,
      "logits/rejected": -1.3937865495681763,
      "logps/chosen": -1.0526535511016846,
      "logps/rejected": -3.306544780731201,
      "loss": -0.1278,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.0526535511016846,
      "rewards/margins": 2.2538914680480957,
      "rewards/rejected": -3.306544780731201,
      "step": 375
    },
    {
      "epoch": 0.7956032452237635,
      "grad_norm": 14.808944976338482,
      "learning_rate": 6.046442623320145e-08,
      "logits/chosen": -1.7755823135375977,
      "logits/rejected": -1.715908408164978,
      "logps/chosen": -1.3490417003631592,
      "logps/rejected": -1.8757483959197998,
      "loss": -0.1173,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.3490417003631592,
      "rewards/margins": 0.5267067551612854,
      "rewards/rejected": -1.8757483959197998,
      "step": 380
    },
    {
      "epoch": 0.8060717089767077,
      "grad_norm": 37.744351087223556,
      "learning_rate": 5.4623689209832484e-08,
      "logits/chosen": -1.621198296546936,
      "logits/rejected": -1.5765188932418823,
      "logps/chosen": -1.1883819103240967,
      "logps/rejected": -2.3331074714660645,
      "loss": -0.1254,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.1883819103240967,
      "rewards/margins": 1.1447254419326782,
      "rewards/rejected": -2.3331074714660645,
      "step": 385
    },
    {
      "epoch": 0.816540172729652,
      "grad_norm": 40.457315431893065,
      "learning_rate": 4.904486005914027e-08,
      "logits/chosen": -1.804915189743042,
      "logits/rejected": -1.42214834690094,
      "logps/chosen": -1.3127548694610596,
      "logps/rejected": -2.009843349456787,
      "loss": -0.1109,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.3127548694610596,
      "rewards/margins": 0.697088360786438,
      "rewards/rejected": -2.009843349456787,
      "step": 390
    },
    {
      "epoch": 0.8270086364825961,
      "grad_norm": 24.94087598352193,
      "learning_rate": 4.373541737087263e-08,
      "logits/chosen": -1.7377560138702393,
      "logits/rejected": -1.3866163492202759,
      "logps/chosen": -1.1465688943862915,
      "logps/rejected": -2.7002861499786377,
      "loss": -0.12,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.1465688943862915,
      "rewards/margins": 1.5537172555923462,
      "rewards/rejected": -2.7002861499786377,
      "step": 395
    },
    {
      "epoch": 0.8374771002355405,
      "grad_norm": 88.79784088043822,
      "learning_rate": 3.8702478614051345e-08,
      "logits/chosen": -1.7607088088989258,
      "logits/rejected": -1.5547138452529907,
      "logps/chosen": -1.3398617506027222,
      "logps/rejected": -2.071460008621216,
      "loss": -0.1154,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.3398617506027222,
      "rewards/margins": 0.7315981388092041,
      "rewards/rejected": -2.071460008621216,
      "step": 400
    },
    {
      "epoch": 0.8479455639884846,
      "grad_norm": 9.248843953744515,
      "learning_rate": 3.3952790595787986e-08,
      "logits/chosen": -1.6915082931518555,
      "logits/rejected": -1.4412751197814941,
      "logps/chosen": -1.4074960947036743,
      "logps/rejected": -2.708440065383911,
      "loss": -0.1226,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.4074960947036743,
      "rewards/margins": 1.3009440898895264,
      "rewards/rejected": -2.708440065383911,
      "step": 405
    },
    {
      "epoch": 0.8584140277414289,
      "grad_norm": 19.94643713892694,
      "learning_rate": 2.9492720416985e-08,
      "logits/chosen": -1.532260537147522,
      "logits/rejected": -1.374741554260254,
      "logps/chosen": -1.6915867328643799,
      "logps/rejected": -3.1600654125213623,
      "loss": -0.137,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.6915867328643799,
      "rewards/margins": 1.4684789180755615,
      "rewards/rejected": -3.1600654125213623,
      "step": 410
    },
    {
      "epoch": 0.8688824914943732,
      "grad_norm": 130.75278775438778,
      "learning_rate": 2.5328246937043525e-08,
      "logits/chosen": -1.5330517292022705,
      "logits/rejected": -1.3144365549087524,
      "logps/chosen": -1.8018486499786377,
      "logps/rejected": -2.8836231231689453,
      "loss": -0.1262,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.8018486499786377,
      "rewards/margins": 1.0817744731903076,
      "rewards/rejected": -2.8836231231689453,
      "step": 415
    },
    {
      "epoch": 0.8793509552473174,
      "grad_norm": 25.478377748856754,
      "learning_rate": 2.1464952759020856e-08,
      "logits/chosen": -1.623173475265503,
      "logits/rejected": -1.3844332695007324,
      "logps/chosen": -1.4715114831924438,
      "logps/rejected": -2.6988463401794434,
      "loss": -0.1389,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.4715114831924438,
      "rewards/margins": 1.2273352146148682,
      "rewards/rejected": -2.6988463401794434,
      "step": 420
    },
    {
      "epoch": 0.8898194190002617,
      "grad_norm": 44.22845937144706,
      "learning_rate": 1.7908016745981856e-08,
      "logits/chosen": -1.6746991872787476,
      "logits/rejected": -1.4371649026870728,
      "logps/chosen": -1.6984422206878662,
      "logps/rejected": -2.9744515419006348,
      "loss": -0.1342,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.6984422206878662,
      "rewards/margins": 1.2760093212127686,
      "rewards/rejected": -2.9744515419006348,
      "step": 425
    },
    {
      "epoch": 0.9002878827532059,
      "grad_norm": 10.062945084378958,
      "learning_rate": 1.4662207078575684e-08,
      "logits/chosen": -1.5444406270980835,
      "logits/rejected": -1.2706645727157593,
      "logps/chosen": -1.5169013738632202,
      "logps/rejected": -3.1792664527893066,
      "loss": -0.1458,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.5169013738632202,
      "rewards/margins": 1.6623647212982178,
      "rewards/rejected": -3.1792664527893066,
      "step": 430
    },
    {
      "epoch": 0.9107563465061502,
      "grad_norm": 26.84204726450129,
      "learning_rate": 1.1731874863145142e-08,
      "logits/chosen": -1.4832799434661865,
      "logits/rejected": -1.6065374612808228,
      "logps/chosen": -1.2497235536575317,
      "logps/rejected": -2.3406825065612793,
      "loss": -0.1329,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.2497235536575317,
      "rewards/margins": 1.0909589529037476,
      "rewards/rejected": -2.3406825065612793,
      "step": 435
    },
    {
      "epoch": 0.9212248102590945,
      "grad_norm": 14.511762413527599,
      "learning_rate": 9.12094829893642e-09,
      "logits/chosen": -1.6549100875854492,
      "logits/rejected": -1.6303857564926147,
      "logps/chosen": -1.6263110637664795,
      "logps/rejected": -2.7538211345672607,
      "loss": -0.1285,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.6263110637664795,
      "rewards/margins": 1.1275101900100708,
      "rewards/rejected": -2.7538211345672607,
      "step": 440
    },
    {
      "epoch": 0.9316932740120387,
      "grad_norm": 115.57689083092143,
      "learning_rate": 6.832927412229017e-09,
      "logits/chosen": -1.6242246627807617,
      "logits/rejected": -1.3819233179092407,
      "logps/chosen": -1.624022126197815,
      "logps/rejected": -2.9669742584228516,
      "loss": -0.1358,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.624022126197815,
      "rewards/margins": 1.3429521322250366,
      "rewards/rejected": -2.9669742584228516,
      "step": 445
    },
    {
      "epoch": 0.942161737764983,
      "grad_norm": 34.008733673808756,
      "learning_rate": 4.8708793644441086e-09,
      "logits/chosen": -1.7152725458145142,
      "logits/rejected": -1.4042075872421265,
      "logps/chosen": -1.1938213109970093,
      "logps/rejected": -2.202075958251953,
      "loss": -0.1383,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.1938213109970093,
      "rewards/margins": 1.0082545280456543,
      "rewards/rejected": -2.202075958251953,
      "step": 450
    },
    {
      "epoch": 0.9526302015179272,
      "grad_norm": 17.121700600376354,
      "learning_rate": 3.2374343405217884e-09,
      "logits/chosen": -1.6548484563827515,
      "logits/rejected": -1.4442580938339233,
      "logps/chosen": -1.8814382553100586,
      "logps/rejected": -3.200958251953125,
      "loss": -0.1119,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.8814382553100586,
      "rewards/margins": 1.3195202350616455,
      "rewards/rejected": -3.200958251953125,
      "step": 455
    },
    {
      "epoch": 0.9630986652708715,
      "grad_norm": 71.31324195680072,
      "learning_rate": 1.9347820230782295e-09,
      "logits/chosen": -1.741633415222168,
      "logits/rejected": -1.2778360843658447,
      "logps/chosen": -1.1031286716461182,
      "logps/rejected": -2.995805025100708,
      "loss": -0.1387,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.1031286716461182,
      "rewards/margins": 1.892676591873169,
      "rewards/rejected": -2.995805025100708,
      "step": 460
    },
    {
      "epoch": 0.9735671290238157,
      "grad_norm": 17.102606051768348,
      "learning_rate": 9.64668657069706e-10,
      "logits/chosen": -1.6080182790756226,
      "logits/rejected": -1.2144018411636353,
      "logps/chosen": -1.5425249338150024,
      "logps/rejected": -2.9966964721679688,
      "loss": -0.1402,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.5425249338150024,
      "rewards/margins": 1.454171895980835,
      "rewards/rejected": -2.9966964721679688,
      "step": 465
    },
    {
      "epoch": 0.98403559277676,
      "grad_norm": 13.352840323224417,
      "learning_rate": 3.2839470889836627e-10,
      "logits/chosen": -1.6037025451660156,
      "logits/rejected": -1.2192766666412354,
      "logps/chosen": -1.357714295387268,
      "logps/rejected": -3.102246046066284,
      "loss": -0.1326,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.357714295387268,
      "rewards/margins": 1.7445319890975952,
      "rewards/rejected": -3.102246046066284,
      "step": 470
    },
    {
      "epoch": 0.9945040565297043,
      "grad_norm": 57.54326325826418,
      "learning_rate": 2.6813123097352287e-11,
      "logits/chosen": -1.570783257484436,
      "logits/rejected": -1.3488304615020752,
      "logps/chosen": -1.446059226989746,
      "logps/rejected": -3.1210341453552246,
      "loss": -0.1382,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.446059226989746,
      "rewards/margins": 1.674975037574768,
      "rewards/rejected": -3.1210341453552246,
      "step": 475
    },
    {
      "epoch": 0.998691442030882,
      "step": 477,
      "total_flos": 0.0,
      "train_loss": -0.10643900037436616,
      "train_runtime": 11655.0761,
      "train_samples_per_second": 5.245,
      "train_steps_per_second": 0.041
    }
  ],
  "logging_steps": 5,
  "max_steps": 477,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 239,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}