trainer_state.json

{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 1911,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 2.6041666666666664e-09,
      "logits/chosen": -2.4222943782806396,
      "logits/rejected": -2.300191879272461,
      "logps/chosen": -242.6232147216797,
      "logps/rejected": -173.6947784423828,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.01,
      "learning_rate": 2.6041666666666667e-08,
      "logits/chosen": -2.580005645751953,
      "logits/rejected": -2.4936344623565674,
      "logps/chosen": -268.85345458984375,
      "logps/rejected": -210.68524169921875,
      "loss": 0.6906,
      "rewards/accuracies": 0.5833333134651184,
      "rewards/chosen": 0.004795679822564125,
      "rewards/margins": 0.013473947532474995,
      "rewards/rejected": -0.008678268641233444,
      "step": 10
    },
    {
      "epoch": 0.01,
      "learning_rate": 5.208333333333333e-08,
      "logits/chosen": -2.693087339401245,
      "logits/rejected": -2.5713565349578857,
      "logps/chosen": -301.3067321777344,
      "logps/rejected": -251.20126342773438,
      "loss": 0.6852,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.0008392825839109719,
      "rewards/margins": 0.013224393129348755,
      "rewards/rejected": -0.012385109439492226,
      "step": 20
    },
    {
      "epoch": 0.02,
      "learning_rate": 7.812499999999999e-08,
      "logits/chosen": -2.725404977798462,
      "logits/rejected": -2.4892578125,
      "logps/chosen": -297.3845520019531,
      "logps/rejected": -222.6193389892578,
      "loss": 0.6594,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.059859294444322586,
      "rewards/margins": 0.09642773121595383,
      "rewards/rejected": -0.03656843304634094,
      "step": 30
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.0416666666666667e-07,
      "logits/chosen": -2.66707706451416,
      "logits/rejected": -2.561459541320801,
      "logps/chosen": -258.4730224609375,
      "logps/rejected": -230.8173065185547,
      "loss": 0.6212,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.16431955993175507,
      "rewards/margins": 0.18100649118423462,
      "rewards/rejected": -0.016686933115124702,
      "step": 40
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.3020833333333334e-07,
      "logits/chosen": -2.6738762855529785,
      "logits/rejected": -2.5567877292633057,
      "logps/chosen": -321.64422607421875,
      "logps/rejected": -266.5691833496094,
      "loss": 0.5692,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.4113697111606598,
      "rewards/margins": 0.3625262379646301,
      "rewards/rejected": 0.048843465745449066,
      "step": 50
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.5624999999999999e-07,
      "logits/chosen": -2.6034302711486816,
      "logits/rejected": -2.437656879425049,
      "logps/chosen": -301.7183837890625,
      "logps/rejected": -295.6407165527344,
      "loss": 0.5304,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.41556516289711,
      "rewards/margins": 0.5151094198226929,
      "rewards/rejected": -0.09954418987035751,
      "step": 60
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.8229166666666666e-07,
      "logits/chosen": -2.563300132751465,
      "logits/rejected": -2.472400426864624,
      "logps/chosen": -265.8623046875,
      "logps/rejected": -256.015869140625,
      "loss": 0.5141,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.4251571297645569,
      "rewards/margins": 0.5559587478637695,
      "rewards/rejected": -0.13080164790153503,
      "step": 70
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.0833333333333333e-07,
      "logits/chosen": -2.6306231021881104,
      "logits/rejected": -2.4977340698242188,
      "logps/chosen": -248.59957885742188,
      "logps/rejected": -230.89956665039062,
      "loss": 0.4591,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.5105770826339722,
      "rewards/margins": 0.6713550686836243,
      "rewards/rejected": -0.1607779711484909,
      "step": 80
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.3437499999999998e-07,
      "logits/chosen": -2.5985584259033203,
      "logits/rejected": -2.4967856407165527,
      "logps/chosen": -270.8584289550781,
      "logps/rejected": -265.82183837890625,
      "loss": 0.4679,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3090444505214691,
      "rewards/margins": 0.8424568176269531,
      "rewards/rejected": -0.5334123373031616,
      "step": 90
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.604166666666667e-07,
      "logits/chosen": -2.5061540603637695,
      "logits/rejected": -2.3507046699523926,
      "logps/chosen": -235.2123565673828,
      "logps/rejected": -198.79722595214844,
      "loss": 0.4467,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.6367200613021851,
      "rewards/margins": 1.0976603031158447,
      "rewards/rejected": -0.4609402120113373,
      "step": 100
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.864583333333333e-07,
      "logits/chosen": -2.569188117980957,
      "logits/rejected": -2.4755890369415283,
      "logps/chosen": -298.9624938964844,
      "logps/rejected": -258.39947509765625,
      "loss": 0.4311,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.614601731300354,
      "rewards/margins": 1.216589331626892,
      "rewards/rejected": -0.6019877195358276,
      "step": 110
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.1249999999999997e-07,
      "logits/chosen": -2.5569252967834473,
      "logits/rejected": -2.398772716522217,
      "logps/chosen": -280.63787841796875,
      "logps/rejected": -212.21267700195312,
      "loss": 0.4719,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.8052973747253418,
      "rewards/margins": 1.0679086446762085,
      "rewards/rejected": -0.26261135935783386,
      "step": 120
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.3854166666666667e-07,
      "logits/chosen": -2.6138534545898438,
      "logits/rejected": -2.3813750743865967,
      "logps/chosen": -311.6466064453125,
      "logps/rejected": -238.57833862304688,
      "loss": 0.4179,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 0.803765594959259,
      "rewards/margins": 1.384441614151001,
      "rewards/rejected": -0.5806761980056763,
      "step": 130
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.645833333333333e-07,
      "logits/chosen": -2.620131015777588,
      "logits/rejected": -2.4834771156311035,
      "logps/chosen": -280.6990051269531,
      "logps/rejected": -241.31381225585938,
      "loss": 0.4193,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.6271090507507324,
      "rewards/margins": 1.3870524168014526,
      "rewards/rejected": -0.7599433660507202,
      "step": 140
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.9062499999999997e-07,
      "logits/chosen": -2.4387025833129883,
      "logits/rejected": -2.3931989669799805,
      "logps/chosen": -277.7287292480469,
      "logps/rejected": -276.62030029296875,
      "loss": 0.3709,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.10160081088542938,
      "rewards/margins": 1.4848428964614868,
      "rewards/rejected": -1.3832420110702515,
      "step": 150
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.1666666666666667e-07,
      "logits/chosen": -2.5423970222473145,
      "logits/rejected": -2.4555859565734863,
      "logps/chosen": -271.20391845703125,
      "logps/rejected": -272.84075927734375,
      "loss": 0.3887,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.8339279294013977,
      "rewards/margins": 2.0397675037384033,
      "rewards/rejected": -1.2058393955230713,
      "step": 160
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.427083333333333e-07,
      "logits/chosen": -2.5736887454986572,
      "logits/rejected": -2.423067331314087,
      "logps/chosen": -296.3034973144531,
      "logps/rejected": -254.79971313476562,
      "loss": 0.4365,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.1544642299413681,
      "rewards/margins": 1.3991954326629639,
      "rewards/rejected": -1.5536595582962036,
      "step": 170
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.6874999999999996e-07,
      "logits/chosen": -2.571223735809326,
      "logits/rejected": -2.438267946243286,
      "logps/chosen": -305.3669128417969,
      "logps/rejected": -290.1111145019531,
      "loss": 0.3981,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.318599134683609,
      "rewards/margins": 1.9614486694335938,
      "rewards/rejected": -1.642849326133728,
      "step": 180
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.947916666666667e-07,
      "logits/chosen": -2.4852206707000732,
      "logits/rejected": -2.301888942718506,
      "logps/chosen": -280.9864196777344,
      "logps/rejected": -234.48117065429688,
      "loss": 0.3926,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.24488838016986847,
      "rewards/margins": 2.234631061553955,
      "rewards/rejected": -1.9897428750991821,
      "step": 190
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.976730657358929e-07,
      "logits/chosen": -2.3834030628204346,
      "logits/rejected": -2.3198955059051514,
      "logps/chosen": -260.1601257324219,
      "logps/rejected": -241.60348510742188,
      "loss": 0.4712,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.1966917514801025,
      "rewards/margins": 1.3776226043701172,
      "rewards/rejected": -2.5743143558502197,
      "step": 200
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.947643979057592e-07,
      "logits/chosen": -2.4042606353759766,
      "logits/rejected": -2.3647255897521973,
      "logps/chosen": -265.72344970703125,
      "logps/rejected": -300.393798828125,
      "loss": 0.405,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 0.035271644592285156,
      "rewards/margins": 1.8820867538452148,
      "rewards/rejected": -1.8468151092529297,
      "step": 210
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.918557300756254e-07,
      "logits/chosen": -2.2479195594787598,
      "logits/rejected": -2.126936435699463,
      "logps/chosen": -286.0814514160156,
      "logps/rejected": -230.5002899169922,
      "loss": 0.4365,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.2404344528913498,
      "rewards/margins": 1.7336857318878174,
      "rewards/rejected": -1.9741201400756836,
      "step": 220
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.889470622454916e-07,
      "logits/chosen": -2.402921199798584,
      "logits/rejected": -2.213413715362549,
      "logps/chosen": -304.3617248535156,
      "logps/rejected": -279.91497802734375,
      "loss": 0.4291,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.006418424658477306,
      "rewards/margins": 2.0425734519958496,
      "rewards/rejected": -2.0489919185638428,
      "step": 230
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.860383944153578e-07,
      "logits/chosen": -2.4241273403167725,
      "logits/rejected": -2.272402048110962,
      "logps/chosen": -287.0025939941406,
      "logps/rejected": -263.2986145019531,
      "loss": 0.381,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.207146555185318,
      "rewards/margins": 1.902012586593628,
      "rewards/rejected": -2.109158992767334,
      "step": 240
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.83129726585224e-07,
      "logits/chosen": -2.429861545562744,
      "logits/rejected": -2.3017280101776123,
      "logps/chosen": -257.54071044921875,
      "logps/rejected": -265.039794921875,
      "loss": 0.3962,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.5596320629119873,
      "rewards/margins": 1.9522535800933838,
      "rewards/rejected": -2.511885404586792,
      "step": 250
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.802210587550902e-07,
      "logits/chosen": -2.3487319946289062,
      "logits/rejected": -2.2305383682250977,
      "logps/chosen": -304.7814025878906,
      "logps/rejected": -278.561279296875,
      "loss": 0.3971,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.19774925708770752,
      "rewards/margins": 1.9618850946426392,
      "rewards/rejected": -2.1596343517303467,
      "step": 260
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.773123909249563e-07,
      "logits/chosen": -2.4337475299835205,
      "logits/rejected": -2.2924251556396484,
      "logps/chosen": -274.05218505859375,
      "logps/rejected": -279.85198974609375,
      "loss": 0.4099,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.34399348497390747,
      "rewards/margins": 3.3459434509277344,
      "rewards/rejected": -3.689936399459839,
      "step": 270
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.7440372309482255e-07,
      "logits/chosen": -2.390204906463623,
      "logits/rejected": -2.249260663986206,
      "logps/chosen": -292.68243408203125,
      "logps/rejected": -268.0016174316406,
      "loss": 0.6235,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.024298882111907005,
      "rewards/margins": 2.0128695964813232,
      "rewards/rejected": -2.037168502807617,
      "step": 280
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.7149505526468876e-07,
      "logits/chosen": -2.356079578399658,
      "logits/rejected": -2.1153652667999268,
      "logps/chosen": -292.0235595703125,
      "logps/rejected": -256.1002502441406,
      "loss": 0.322,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.1279529184103012,
      "rewards/margins": 2.2269229888916016,
      "rewards/rejected": -2.0989699363708496,
      "step": 290
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.6858638743455497e-07,
      "logits/chosen": -2.472639322280884,
      "logits/rejected": -2.3683533668518066,
      "logps/chosen": -338.8008728027344,
      "logps/rejected": -282.82843017578125,
      "loss": 0.4142,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.14725220203399658,
      "rewards/margins": 1.3299568891525269,
      "rewards/rejected": -1.4772090911865234,
      "step": 300
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.656777196044212e-07,
      "logits/chosen": -2.4232888221740723,
      "logits/rejected": -2.2172470092773438,
      "logps/chosen": -296.7711181640625,
      "logps/rejected": -261.01995849609375,
      "loss": 0.4658,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.06699763238430023,
      "rewards/margins": 2.3037867546081543,
      "rewards/rejected": -2.2367894649505615,
      "step": 310
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.627690517742874e-07,
      "logits/chosen": -2.327702283859253,
      "logits/rejected": -2.200692892074585,
      "logps/chosen": -241.1982421875,
      "logps/rejected": -220.4070281982422,
      "loss": 0.3734,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.3033676743507385,
      "rewards/margins": 2.032435417175293,
      "rewards/rejected": -1.7290678024291992,
      "step": 320
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.5986038394415354e-07,
      "logits/chosen": -2.321364641189575,
      "logits/rejected": -2.122469902038574,
      "logps/chosen": -284.7498779296875,
      "logps/rejected": -232.0214385986328,
      "loss": 0.6356,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.3031235337257385,
      "rewards/margins": 2.075382709503174,
      "rewards/rejected": -2.3785061836242676,
      "step": 330
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.569517161140198e-07,
      "logits/chosen": -2.420304775238037,
      "logits/rejected": -2.304600954055786,
      "logps/chosen": -280.8135070800781,
      "logps/rejected": -293.92681884765625,
      "loss": 0.5416,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.15361396968364716,
      "rewards/margins": 1.9998279809951782,
      "rewards/rejected": -1.846213936805725,
      "step": 340
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.5404304828388595e-07,
      "logits/chosen": -2.367790699005127,
      "logits/rejected": -2.225597381591797,
      "logps/chosen": -298.0318298339844,
      "logps/rejected": -295.94256591796875,
      "loss": 0.3371,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.41088590025901794,
      "rewards/margins": 2.291506767272949,
      "rewards/rejected": -2.702393054962158,
      "step": 350
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.5113438045375216e-07,
      "logits/chosen": -2.25712251663208,
      "logits/rejected": -2.0587408542633057,
      "logps/chosen": -273.820068359375,
      "logps/rejected": -240.53372192382812,
      "loss": 0.3219,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08009258657693863,
      "rewards/margins": 2.750244617462158,
      "rewards/rejected": -2.8303377628326416,
      "step": 360
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.4822571262361837e-07,
      "logits/chosen": -2.2759785652160645,
      "logits/rejected": -2.10341477394104,
      "logps/chosen": -232.6130828857422,
      "logps/rejected": -221.0072784423828,
      "loss": 0.4257,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.17208755016326904,
      "rewards/margins": 2.1418089866638184,
      "rewards/rejected": -2.313896417617798,
      "step": 370
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.453170447934846e-07,
      "logits/chosen": -2.2166106700897217,
      "logits/rejected": -2.0448429584503174,
      "logps/chosen": -300.9383239746094,
      "logps/rejected": -248.05429077148438,
      "loss": 0.3711,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.026472803205251694,
      "rewards/margins": 2.597238779067993,
      "rewards/rejected": -2.623711585998535,
      "step": 380
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.424083769633508e-07,
      "logits/chosen": -2.3724656105041504,
      "logits/rejected": -2.2095799446105957,
      "logps/chosen": -301.7389221191406,
      "logps/rejected": -317.2711181640625,
      "loss": 0.4157,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.7330515384674072,
      "rewards/margins": 2.163984775543213,
      "rewards/rejected": -2.897036552429199,
      "step": 390
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.39499709133217e-07,
      "logits/chosen": -2.490659713745117,
      "logits/rejected": -2.345731496810913,
      "logps/chosen": -308.53558349609375,
      "logps/rejected": -294.06500244140625,
      "loss": 0.4574,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.5682181715965271,
      "rewards/margins": 2.168735980987549,
      "rewards/rejected": -2.7369542121887207,
      "step": 400
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.3659104130308314e-07,
      "logits/chosen": -2.3574600219726562,
      "logits/rejected": -2.245293617248535,
      "logps/chosen": -264.3704833984375,
      "logps/rejected": -270.0927734375,
      "loss": 0.3066,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.676201581954956,
      "rewards/margins": 2.4835641384124756,
      "rewards/rejected": -3.1597657203674316,
      "step": 410
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.336823734729494e-07,
      "logits/chosen": -2.542794704437256,
      "logits/rejected": -2.2256128787994385,
      "logps/chosen": -327.1799621582031,
      "logps/rejected": -270.61065673828125,
      "loss": 0.3053,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.38001373410224915,
      "rewards/margins": 2.6601920127868652,
      "rewards/rejected": -3.040205478668213,
      "step": 420
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.3077370564281556e-07,
      "logits/chosen": -2.393115997314453,
      "logits/rejected": -2.141357421875,
      "logps/chosen": -291.4068908691406,
      "logps/rejected": -254.27560424804688,
      "loss": 0.3113,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.41613179445266724,
      "rewards/margins": 2.6375651359558105,
      "rewards/rejected": -3.053696870803833,
      "step": 430
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.278650378126818e-07,
      "logits/chosen": -2.2429466247558594,
      "logits/rejected": -2.1097540855407715,
      "logps/chosen": -283.1539001464844,
      "logps/rejected": -235.98074340820312,
      "loss": 0.3876,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.28720611333847046,
      "rewards/margins": 2.393641233444214,
      "rewards/rejected": -2.680847406387329,
      "step": 440
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.24956369982548e-07,
      "logits/chosen": -2.3428943157196045,
      "logits/rejected": -2.1891512870788574,
      "logps/chosen": -305.74981689453125,
      "logps/rejected": -277.8802185058594,
      "loss": 0.3838,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.138112872838974,
      "rewards/margins": 2.6869564056396484,
      "rewards/rejected": -2.8250691890716553,
      "step": 450
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.220477021524142e-07,
      "logits/chosen": -2.3777918815612793,
      "logits/rejected": -2.1941428184509277,
      "logps/chosen": -264.9512023925781,
      "logps/rejected": -269.465087890625,
      "loss": 0.3758,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.27588149905204773,
      "rewards/margins": 2.346616506576538,
      "rewards/rejected": -2.622498035430908,
      "step": 460
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.191390343222804e-07,
      "logits/chosen": -2.412323474884033,
      "logits/rejected": -2.234257698059082,
      "logps/chosen": -298.27630615234375,
      "logps/rejected": -266.40460205078125,
      "loss": 0.4628,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.25904446840286255,
      "rewards/margins": 2.111635684967041,
      "rewards/rejected": -2.370680093765259,
      "step": 470
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.162303664921466e-07,
      "logits/chosen": -2.35739803314209,
      "logits/rejected": -2.168689250946045,
      "logps/chosen": -281.678955078125,
      "logps/rejected": -260.5196838378906,
      "loss": 0.3468,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.45242181420326233,
      "rewards/margins": 2.4778125286102295,
      "rewards/rejected": -2.930233955383301,
      "step": 480
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.1332169866201275e-07,
      "logits/chosen": -2.286667585372925,
      "logits/rejected": -2.049354076385498,
      "logps/chosen": -283.5222473144531,
      "logps/rejected": -245.1978759765625,
      "loss": 0.4088,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.19852662086486816,
      "rewards/margins": 2.8068137168884277,
      "rewards/rejected": -2.6082868576049805,
      "step": 490
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.10413030831879e-07,
      "logits/chosen": -2.423564910888672,
      "logits/rejected": -2.1486616134643555,
      "logps/chosen": -315.94171142578125,
      "logps/rejected": -240.4951629638672,
      "loss": 0.3619,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.18976499140262604,
      "rewards/margins": 2.241330146789551,
      "rewards/rejected": -2.4310948848724365,
      "step": 500
    },
    {
      "epoch": 0.26,
      "eval_logits/chosen": -2.3617773056030273,
      "eval_logits/rejected": -2.199384927749634,
      "eval_logps/chosen": -282.76519775390625,
      "eval_logps/rejected": -265.9465637207031,
      "eval_loss": 0.3821512758731842,
      "eval_rewards/accuracies": 0.8650793433189392,
      "eval_rewards/chosen": 0.18425174057483673,
      "eval_rewards/margins": 2.281205654144287,
      "eval_rewards/rejected": -2.096953868865967,
      "eval_runtime": 717.3734,
      "eval_samples_per_second": 2.788,
      "eval_steps_per_second": 0.088,
      "step": 500
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.0750436300174517e-07,
      "logits/chosen": -2.295250415802002,
      "logits/rejected": -2.2096400260925293,
      "logps/chosen": -291.8120422363281,
      "logps/rejected": -260.5565490722656,
      "loss": 0.3481,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.032511450350284576,
      "rewards/margins": 1.999352216720581,
      "rewards/rejected": -2.0318636894226074,
      "step": 510
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.0459569517161143e-07,
      "logits/chosen": -2.3412113189697266,
      "logits/rejected": -2.168757915496826,
      "logps/chosen": -289.66015625,
      "logps/rejected": -284.6536560058594,
      "loss": 0.4214,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.024832209572196007,
      "rewards/margins": 2.256009817123413,
      "rewards/rejected": -2.2808423042297363,
      "step": 520
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.016870273414776e-07,
      "logits/chosen": -2.3721296787261963,
      "logits/rejected": -2.216839551925659,
      "logps/chosen": -246.75814819335938,
      "logps/rejected": -261.1737365722656,
      "loss": 0.3284,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.24080362915992737,
      "rewards/margins": 2.464489459991455,
      "rewards/rejected": -2.7052931785583496,
      "step": 530
    },
    {
      "epoch": 0.28,
      "learning_rate": 3.987783595113438e-07,
      "logits/chosen": -2.3687021732330322,
      "logits/rejected": -2.1532135009765625,
      "logps/chosen": -329.71002197265625,
      "logps/rejected": -260.98931884765625,
      "loss": 0.3913,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.147232323884964,
      "rewards/margins": 2.5936901569366455,
      "rewards/rejected": -2.740922451019287,
      "step": 540
    },
    {
      "epoch": 0.29,
      "learning_rate": 3.9586969168121e-07,
      "logits/chosen": -2.3451223373413086,
      "logits/rejected": -2.1668312549591064,
      "logps/chosen": -337.485595703125,
      "logps/rejected": -296.2718200683594,
      "loss": 0.3295,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.21495577692985535,
      "rewards/margins": 2.442884922027588,
      "rewards/rejected": -2.227928876876831,
      "step": 550
    },
    {
      "epoch": 0.29,
      "learning_rate": 3.929610238510762e-07,
      "logits/chosen": -2.26320481300354,
      "logits/rejected": -2.1082823276519775,
      "logps/chosen": -280.9175720214844,
      "logps/rejected": -272.5048522949219,
      "loss": 0.4014,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.1318025290966034,
      "rewards/margins": 2.3364672660827637,
      "rewards/rejected": -2.204664707183838,
      "step": 560
    },
    {
      "epoch": 0.3,
      "learning_rate": 3.900523560209424e-07,
      "logits/chosen": -2.3491129875183105,
      "logits/rejected": -2.1811981201171875,
      "logps/chosen": -303.96441650390625,
      "logps/rejected": -286.36212158203125,
      "loss": 0.3572,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.1058625802397728,
      "rewards/margins": 2.76606822013855,
      "rewards/rejected": -2.660205364227295,
      "step": 570
    },
    {
      "epoch": 0.3,
      "learning_rate": 3.871436881908086e-07,
      "logits/chosen": -2.396277904510498,
      "logits/rejected": -2.1476378440856934,
      "logps/chosen": -274.61334228515625,
      "logps/rejected": -237.1002960205078,
      "loss": 0.3261,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.008923185057938099,
      "rewards/margins": 2.317211151123047,
      "rewards/rejected": -2.326134443283081,
      "step": 580
    },
    {
      "epoch": 0.31,
      "learning_rate": 3.842350203606748e-07,
      "logits/chosen": -2.30973482131958,
      "logits/rejected": -2.0597481727600098,
      "logps/chosen": -255.37875366210938,
      "logps/rejected": -235.5735626220703,
      "loss": 0.3845,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5673273801803589,
      "rewards/margins": 2.377692222595215,
      "rewards/rejected": -2.945019245147705,
      "step": 590
    },
    {
      "epoch": 0.31,
      "learning_rate": 3.8132635253054103e-07,
      "logits/chosen": -2.2930376529693604,
      "logits/rejected": -2.230609893798828,
      "logps/chosen": -288.5751953125,
      "logps/rejected": -318.2559509277344,
      "loss": 0.354,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.0748153105378151,
      "rewards/margins": 2.7162458896636963,
      "rewards/rejected": -2.7910614013671875,
      "step": 600
    },
    {
      "epoch": 0.32,
      "learning_rate": 3.784176847004072e-07,
      "logits/chosen": -2.3762922286987305,
      "logits/rejected": -2.3032281398773193,
      "logps/chosen": -266.0533447265625,
      "logps/rejected": -305.07257080078125,
      "loss": 0.3032,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.32922041416168213,
      "rewards/margins": 2.986255645751953,
      "rewards/rejected": -3.315476179122925,
      "step": 610
    },
    {
      "epoch": 0.32,
      "learning_rate": 3.755090168702734e-07,
      "logits/chosen": -2.313537120819092,
      "logits/rejected": -2.25026273727417,
      "logps/chosen": -306.7781982421875,
      "logps/rejected": -339.4481506347656,
      "loss": 0.3557,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.6963660717010498,
      "rewards/margins": 2.4231371879577637,
      "rewards/rejected": -3.1195030212402344,
      "step": 620
    },
    {
      "epoch": 0.33,
      "learning_rate": 3.726003490401396e-07,
      "logits/chosen": -2.2563579082489014,
      "logits/rejected": -2.1553292274475098,
      "logps/chosen": -288.6074523925781,
      "logps/rejected": -262.7868957519531,
      "loss": 0.3146,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.44406232237815857,
      "rewards/margins": 3.098081588745117,
      "rewards/rejected": -3.5421440601348877,
      "step": 630
    },
    {
      "epoch": 0.33,
      "learning_rate": 3.696916812100058e-07,
      "logits/chosen": -2.227241277694702,
      "logits/rejected": -2.0303165912628174,
      "logps/chosen": -301.3541259765625,
      "logps/rejected": -224.4669189453125,
      "loss": 0.2901,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.028151262551546097,
      "rewards/margins": 3.3005497455596924,
      "rewards/rejected": -3.2723987102508545,
      "step": 640
    },
    {
      "epoch": 0.34,
      "learning_rate": 3.66783013379872e-07,
      "logits/chosen": -2.342437744140625,
      "logits/rejected": -2.1738288402557373,
      "logps/chosen": -291.0592346191406,
      "logps/rejected": -245.4440155029297,
      "loss": 0.3027,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.018262123689055443,
      "rewards/margins": 3.1664016246795654,
      "rewards/rejected": -3.184663772583008,
      "step": 650
    },
    {
      "epoch": 0.35,
      "learning_rate": 3.6387434554973823e-07,
      "logits/chosen": -2.269165515899658,
      "logits/rejected": -2.113163709640503,
      "logps/chosen": -308.91119384765625,
      "logps/rejected": -277.9391174316406,
      "loss": 0.3326,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.8833945393562317,
      "rewards/margins": 2.456275701522827,
      "rewards/rejected": -3.339670181274414,
      "step": 660
    },
    {
      "epoch": 0.35,
      "learning_rate": 3.609656777196044e-07,
      "logits/chosen": -2.3179445266723633,
      "logits/rejected": -2.203679084777832,
      "logps/chosen": -320.14263916015625,
      "logps/rejected": -286.0535888671875,
      "loss": 0.3371,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.80712890625,
      "rewards/margins": 2.284970760345459,
      "rewards/rejected": -3.09209942817688,
      "step": 670
    },
    {
      "epoch": 0.36,
      "learning_rate": 3.5805700988947064e-07,
      "logits/chosen": -2.2682688236236572,
      "logits/rejected": -2.1642825603485107,
      "logps/chosen": -253.39920043945312,
      "logps/rejected": -242.94921875,
      "loss": 0.3251,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.13453222811222076,
      "rewards/margins": 2.803748369216919,
      "rewards/rejected": -2.9382805824279785,
      "step": 680
    },
    {
      "epoch": 0.36,
      "learning_rate": 3.551483420593368e-07,
      "logits/chosen": -2.312835454940796,
      "logits/rejected": -2.162968158721924,
      "logps/chosen": -321.691162109375,
      "logps/rejected": -285.85479736328125,
      "loss": 0.3158,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.3543187081813812,
      "rewards/margins": 2.611062526702881,
      "rewards/rejected": -2.965381622314453,
      "step": 690
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.5223967422920306e-07,
      "logits/chosen": -2.2975497245788574,
      "logits/rejected": -2.1003618240356445,
      "logps/chosen": -346.7196960449219,
      "logps/rejected": -309.878662109375,
      "loss": 0.3841,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.6169618368148804,
      "rewards/margins": 2.551251173019409,
      "rewards/rejected": -3.168212890625,
      "step": 700
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.493310063990692e-07,
      "logits/chosen": -2.368072032928467,
      "logits/rejected": -2.1905136108398438,
      "logps/chosen": -249.8124542236328,
      "logps/rejected": -255.54690551757812,
      "loss": 0.3306,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10715383291244507,
      "rewards/margins": 3.3935387134552,
      "rewards/rejected": -3.500692367553711,
      "step": 710
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.464223385689354e-07,
      "logits/chosen": -2.3153772354125977,
      "logits/rejected": -2.2586145401000977,
      "logps/chosen": -242.65805053710938,
      "logps/rejected": -254.4687042236328,
      "loss": 0.3583,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.1414814442396164,
      "rewards/margins": 2.854738235473633,
      "rewards/rejected": -2.713256359100342,
      "step": 720
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.4351367073880163e-07,
      "logits/chosen": -2.318925380706787,
      "logits/rejected": -2.2675676345825195,
      "logps/chosen": -257.1809997558594,
      "logps/rejected": -271.678955078125,
      "loss": 0.2901,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.037356503307819366,
      "rewards/margins": 2.468323230743408,
      "rewards/rejected": -2.43096661567688,
      "step": 730
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.4060500290866783e-07,
      "logits/chosen": -2.3555076122283936,
      "logits/rejected": -2.1742148399353027,
      "logps/chosen": -342.202880859375,
      "logps/rejected": -292.80706787109375,
      "loss": 0.271,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.6387542486190796,
      "rewards/margins": 2.901257038116455,
      "rewards/rejected": -3.540011167526245,
      "step": 740
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.37696335078534e-07,
      "logits/chosen": -2.3542723655700684,
      "logits/rejected": -2.159390687942505,
      "logps/chosen": -291.72039794921875,
      "logps/rejected": -241.2707977294922,
      "loss": 0.3751,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.3051406145095825,
      "rewards/margins": 2.620650053024292,
      "rewards/rejected": -2.925791025161743,
      "step": 750
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.3478766724840025e-07,
      "logits/chosen": -2.2971510887145996,
      "logits/rejected": -2.1574819087982178,
      "logps/chosen": -272.34698486328125,
      "logps/rejected": -245.53323364257812,
      "loss": 0.3298,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.02699587121605873,
      "rewards/margins": 2.5259757041931152,
      "rewards/rejected": -2.552971363067627,
      "step": 760
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.318789994182664e-07,
      "logits/chosen": -2.3655948638916016,
      "logits/rejected": -2.1962826251983643,
      "logps/chosen": -256.5421142578125,
      "logps/rejected": -283.6791687011719,
      "loss": 0.3064,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.1341879814863205,
      "rewards/margins": 2.915113925933838,
      "rewards/rejected": -2.78092622756958,
      "step": 770
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.2897033158813266e-07,
      "logits/chosen": -2.2602274417877197,
      "logits/rejected": -2.1949830055236816,
      "logps/chosen": -282.25225830078125,
      "logps/rejected": -250.8028106689453,
      "loss": 0.4087,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.5231667757034302,
      "rewards/margins": 2.3779044151306152,
      "rewards/rejected": -2.901071548461914,
      "step": 780
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.260616637579988e-07,
      "logits/chosen": -2.3795268535614014,
      "logits/rejected": -2.273205041885376,
      "logps/chosen": -300.8672180175781,
      "logps/rejected": -285.58538818359375,
      "loss": 0.3704,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.10278744995594025,
      "rewards/margins": 1.8986539840698242,
      "rewards/rejected": -2.001441478729248,
      "step": 790
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.2315299592786503e-07,
      "logits/chosen": -2.3038344383239746,
      "logits/rejected": -2.1164727210998535,
      "logps/chosen": -280.07183837890625,
      "logps/rejected": -238.2751007080078,
      "loss": 0.3589,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.01835661195218563,
      "rewards/margins": 3.2359859943389893,
      "rewards/rejected": -3.2176296710968018,
      "step": 800
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.2024432809773123e-07,
      "logits/chosen": -2.2451796531677246,
      "logits/rejected": -2.0506649017333984,
      "logps/chosen": -264.8363342285156,
      "logps/rejected": -231.02664184570312,
      "loss": 0.2797,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.010631388053297997,
      "rewards/margins": 3.197535991668701,
      "rewards/rejected": -3.1869049072265625,
      "step": 810
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.1733566026759744e-07,
      "logits/chosen": -2.3159565925598145,
      "logits/rejected": -2.1475565433502197,
      "logps/chosen": -281.8079528808594,
      "logps/rejected": -262.15545654296875,
      "loss": 0.4088,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.36606311798095703,
      "rewards/margins": 2.4230146408081055,
      "rewards/rejected": -2.7890777587890625,
      "step": 820
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.144269924374636e-07,
      "logits/chosen": -2.312073230743408,
      "logits/rejected": -2.1491856575012207,
      "logps/chosen": -263.62823486328125,
      "logps/rejected": -274.19891357421875,
      "loss": 0.374,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.2555127739906311,
      "rewards/margins": 2.482483386993408,
      "rewards/rejected": -2.7379961013793945,
      "step": 830
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.1151832460732986e-07,
      "logits/chosen": -2.22873592376709,
      "logits/rejected": -2.0442733764648438,
      "logps/chosen": -298.53509521484375,
      "logps/rejected": -266.8551025390625,
      "loss": 0.3152,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.09185002744197845,
      "rewards/margins": 3.16621994972229,
      "rewards/rejected": -3.0743699073791504,
      "step": 840
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.08609656777196e-07,
      "logits/chosen": -2.271857738494873,
      "logits/rejected": -2.1578991413116455,
      "logps/chosen": -263.89007568359375,
      "logps/rejected": -253.24179077148438,
      "loss": 0.3368,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.23096470534801483,
      "rewards/margins": 2.82230281829834,
      "rewards/rejected": -3.05326771736145,
      "step": 850
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.0570098894706227e-07,
      "logits/chosen": -2.3438827991485596,
      "logits/rejected": -2.2087767124176025,
      "logps/chosen": -291.0592346191406,
      "logps/rejected": -278.04412841796875,
      "loss": 0.2865,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4321475923061371,
      "rewards/margins": 2.5872695446014404,
      "rewards/rejected": -3.0194175243377686,
      "step": 860
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.0279232111692843e-07,
      "logits/chosen": -2.15918231010437,
      "logits/rejected": -2.081557035446167,
      "logps/chosen": -260.660888671875,
      "logps/rejected": -263.269287109375,
      "loss": 0.3036,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07870881259441376,
      "rewards/margins": 3.0273680686950684,
      "rewards/rejected": -3.106077194213867,
      "step": 870
    },
    {
      "epoch": 0.46,
      "learning_rate": 2.9988365328679463e-07,
      "logits/chosen": -2.30003023147583,
      "logits/rejected": -2.122143507003784,
      "logps/chosen": -305.5731201171875,
      "logps/rejected": -266.93487548828125,
      "loss": 0.3189,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.4314422011375427,
      "rewards/margins": 3.166658401489258,
      "rewards/rejected": -3.5981006622314453,
      "step": 880
    },
    {
      "epoch": 0.47,
      "learning_rate": 2.9697498545666084e-07,
      "logits/chosen": -2.2059383392333984,
      "logits/rejected": -1.9731464385986328,
      "logps/chosen": -342.48468017578125,
      "logps/rejected": -278.19580078125,
      "loss": 0.3215,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.6458533406257629,
      "rewards/margins": 3.3348402976989746,
      "rewards/rejected": -3.980693817138672,
      "step": 890
    },
    {
      "epoch": 0.47,
      "learning_rate": 2.9406631762652705e-07,
      "logits/chosen": -2.264359951019287,
      "logits/rejected": -2.0387301445007324,
      "logps/chosen": -330.88446044921875,
      "logps/rejected": -297.0821228027344,
      "loss": 0.2575,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5124462246894836,
      "rewards/margins": 3.263385772705078,
      "rewards/rejected": -3.775831937789917,
      "step": 900
    },
    {
      "epoch": 0.48,
      "learning_rate": 2.9115764979639326e-07,
      "logits/chosen": -2.270777463912964,
      "logits/rejected": -2.169574022293091,
      "logps/chosen": -307.5442810058594,
      "logps/rejected": -281.299072265625,
      "loss": 0.2926,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.295340895652771,
      "rewards/margins": 3.029505491256714,
      "rewards/rejected": -3.3248462677001953,
      "step": 910
    },
    {
      "epoch": 0.48,
      "learning_rate": 2.8824898196625947e-07,
      "logits/chosen": -2.193376302719116,
      "logits/rejected": -1.9907703399658203,
      "logps/chosen": -290.02862548828125,
      "logps/rejected": -275.67919921875,
      "loss": 0.2917,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.3032880127429962,
      "rewards/margins": 3.0413851737976074,
      "rewards/rejected": -3.3446731567382812,
      "step": 920
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.853403141361256e-07,
      "logits/chosen": -2.1926026344299316,
      "logits/rejected": -1.989322304725647,
      "logps/chosen": -297.0482482910156,
      "logps/rejected": -234.0426788330078,
      "loss": 0.3039,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.6774950623512268,
      "rewards/margins": 2.4298861026763916,
      "rewards/rejected": -3.1073811054229736,
      "step": 930
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.824316463059919e-07,
      "logits/chosen": -2.2069790363311768,
      "logits/rejected": -2.064375877380371,
      "logps/chosen": -295.0317077636719,
      "logps/rejected": -293.92242431640625,
      "loss": 0.3263,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.0432120561599731,
      "rewards/margins": 3.1925106048583984,
      "rewards/rejected": -4.235722541809082,
      "step": 940
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.7952297847585803e-07,
      "logits/chosen": -2.217855930328369,
      "logits/rejected": -2.084010601043701,
      "logps/chosen": -272.4216613769531,
      "logps/rejected": -253.54348754882812,
      "loss": 0.3049,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.8211439847946167,
      "rewards/margins": 2.7119455337524414,
      "rewards/rejected": -3.5330893993377686,
      "step": 950
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.766143106457243e-07,
      "logits/chosen": -2.1376349925994873,
      "logits/rejected": -2.015172004699707,
      "logps/chosen": -259.86810302734375,
      "logps/rejected": -262.43756103515625,
      "loss": 0.3498,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.7693565487861633,
      "rewards/margins": 2.9703726768493652,
      "rewards/rejected": -3.739729404449463,
      "step": 960
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.7370564281559045e-07,
      "logits/chosen": -2.34848952293396,
      "logits/rejected": -2.0991873741149902,
      "logps/chosen": -343.0187683105469,
      "logps/rejected": -266.3363342285156,
      "loss": 0.3301,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.8317963480949402,
      "rewards/margins": 2.7026162147521973,
      "rewards/rejected": -3.534412384033203,
      "step": 970
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.7079697498545666e-07,
      "logits/chosen": -2.2526087760925293,
      "logits/rejected": -2.0652668476104736,
      "logps/chosen": -316.2718811035156,
      "logps/rejected": -283.3409729003906,
      "loss": 0.2974,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.2208749055862427,
      "rewards/margins": 3.0918426513671875,
      "rewards/rejected": -4.312718391418457,
      "step": 980
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.6788830715532287e-07,
      "logits/chosen": -2.316199541091919,
      "logits/rejected": -2.1947271823883057,
      "logps/chosen": -274.7453308105469,
      "logps/rejected": -276.6575927734375,
      "loss": 0.4123,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3303449153900146,
      "rewards/margins": 2.126737117767334,
      "rewards/rejected": -3.4570820331573486,
      "step": 990
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.6497963932518907e-07,
      "logits/chosen": -2.1468493938446045,
      "logits/rejected": -2.0539629459381104,
      "logps/chosen": -293.9585266113281,
      "logps/rejected": -318.6316223144531,
      "loss": 0.396,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.9897568821907043,
      "rewards/margins": 2.690471649169922,
      "rewards/rejected": -3.6802284717559814,
      "step": 1000
    },
    {
      "epoch": 0.52,
      "eval_logits/chosen": -2.2926905155181885,
      "eval_logits/rejected": -2.133467435836792,
      "eval_logps/chosen": -292.16717529296875,
      "eval_logps/rejected": -277.26959228515625,
      "eval_loss": 0.37469375133514404,
      "eval_rewards/accuracies": 0.8730158805847168,
      "eval_rewards/chosen": -0.7559432983398438,
      "eval_rewards/margins": 2.473315715789795,
      "eval_rewards/rejected": -3.2292590141296387,
      "eval_runtime": 715.3867,
      "eval_samples_per_second": 2.796,
      "eval_steps_per_second": 0.088,
      "step": 1000
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6207097149505523e-07,
      "logits/chosen": -2.3242342472076416,
      "logits/rejected": -2.152332067489624,
      "logps/chosen": -317.60345458984375,
      "logps/rejected": -260.40057373046875,
      "loss": 0.3001,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.3242790102958679,
      "rewards/margins": 2.7641217708587646,
      "rewards/rejected": -3.088400363922119,
      "step": 1010
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.591623036649215e-07,
      "logits/chosen": -2.2785840034484863,
      "logits/rejected": -2.119656562805176,
      "logps/chosen": -301.8367614746094,
      "logps/rejected": -281.8397521972656,
      "loss": 0.2918,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5784929394721985,
      "rewards/margins": 2.6694836616516113,
      "rewards/rejected": -3.247976779937744,
      "step": 1020
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5625363583478764e-07,
      "logits/chosen": -2.2209084033966064,
      "logits/rejected": -2.115307569503784,
      "logps/chosen": -272.38092041015625,
      "logps/rejected": -277.6880187988281,
      "loss": 0.2436,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6559833288192749,
      "rewards/margins": 2.454627513885498,
      "rewards/rejected": -3.1106107234954834,
      "step": 1030
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.533449680046539e-07,
      "logits/chosen": -2.2398033142089844,
      "logits/rejected": -2.1163482666015625,
      "logps/chosen": -284.6607360839844,
      "logps/rejected": -289.515625,
      "loss": 0.3228,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.2610655725002289,
      "rewards/margins": 3.0918660163879395,
      "rewards/rejected": -3.352931261062622,
      "step": 1040
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.5043630017452006e-07,
      "logits/chosen": -2.2711968421936035,
      "logits/rejected": -2.1295342445373535,
      "logps/chosen": -284.3731689453125,
      "logps/rejected": -258.94189453125,
      "loss": 0.3527,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.4090282917022705,
      "rewards/margins": 2.8344674110412598,
      "rewards/rejected": -3.2434959411621094,
      "step": 1050
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.4752763234438627e-07,
      "logits/chosen": -2.252903699874878,
      "logits/rejected": -2.149430274963379,
      "logps/chosen": -246.1130828857422,
      "logps/rejected": -267.55377197265625,
      "loss": 0.3028,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.19610390067100525,
      "rewards/margins": 2.8517351150512695,
      "rewards/rejected": -3.0478389263153076,
      "step": 1060
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.4461896451425247e-07,
      "logits/chosen": -2.221062183380127,
      "logits/rejected": -2.1321158409118652,
      "logps/chosen": -254.9988250732422,
      "logps/rejected": -250.8503875732422,
      "loss": 0.3461,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.6029731035232544,
      "rewards/margins": 2.7440178394317627,
      "rewards/rejected": -3.3469905853271484,
      "step": 1070
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.417102966841187e-07,
      "logits/chosen": -2.27197527885437,
      "logits/rejected": -2.067380428314209,
      "logps/chosen": -265.71539306640625,
      "logps/rejected": -266.55487060546875,
      "loss": 0.293,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.19143518805503845,
      "rewards/margins": 3.0950098037719727,
      "rewards/rejected": -3.286444902420044,
      "step": 1080
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3880162885398483e-07,
      "logits/chosen": -2.286116600036621,
      "logits/rejected": -2.009376287460327,
      "logps/chosen": -283.79241943359375,
      "logps/rejected": -210.90280151367188,
      "loss": 0.3128,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3403369188308716,
      "rewards/margins": 3.371168613433838,
      "rewards/rejected": -3.71150541305542,
      "step": 1090
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.3589296102385107e-07,
      "logits/chosen": -2.3620998859405518,
      "logits/rejected": -2.1931567192077637,
      "logps/chosen": -285.1781921386719,
      "logps/rejected": -266.1251220703125,
      "loss": 0.3443,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.029187727719545364,
      "rewards/margins": 2.814174175262451,
      "rewards/rejected": -2.8433618545532227,
      "step": 1100
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.3298429319371725e-07,
      "logits/chosen": -2.351217746734619,
      "logits/rejected": -2.1239571571350098,
      "logps/chosen": -285.3910217285156,
      "logps/rejected": -278.2947082519531,
      "loss": 0.3532,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.21914640069007874,
      "rewards/margins": 2.5958099365234375,
      "rewards/rejected": -2.8149561882019043,
      "step": 1110
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.3007562536358346e-07,
      "logits/chosen": -2.3129007816314697,
      "logits/rejected": -2.1005890369415283,
      "logps/chosen": -281.8247985839844,
      "logps/rejected": -266.05230712890625,
      "loss": 0.4081,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.002577108098194003,
      "rewards/margins": 2.926461935043335,
      "rewards/rejected": -2.929039239883423,
      "step": 1120
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.2716695753344967e-07,
      "logits/chosen": -2.3015172481536865,
      "logits/rejected": -2.125819444656372,
      "logps/chosen": -280.98382568359375,
      "logps/rejected": -234.7398681640625,
      "loss": 0.3517,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.1859440803527832,
      "rewards/margins": 2.21616268157959,
      "rewards/rejected": -2.402106761932373,
      "step": 1130
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.2425828970331587e-07,
      "logits/chosen": -2.346259355545044,
      "logits/rejected": -2.2613463401794434,
      "logps/chosen": -250.6602325439453,
      "logps/rejected": -252.2750244140625,
      "loss": 0.3576,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09936543554067612,
      "rewards/margins": 2.3562135696411133,
      "rewards/rejected": -2.4555790424346924,
      "step": 1140
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.2134962187318208e-07,
      "logits/chosen": -2.2582526206970215,
      "logits/rejected": -2.185497283935547,
      "logps/chosen": -289.13775634765625,
      "logps/rejected": -283.5538330078125,
      "loss": 0.3806,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.5999955534934998,
      "rewards/margins": 2.2260944843292236,
      "rewards/rejected": -2.82608962059021,
      "step": 1150
    },
    {
      "epoch": 0.61,
      "learning_rate": 2.1844095404304826e-07,
      "logits/chosen": -2.275233745574951,
      "logits/rejected": -2.1149070262908936,
      "logps/chosen": -315.2784423828125,
      "logps/rejected": -286.3872375488281,
      "loss": 0.3783,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.8392481803894043,
      "rewards/margins": 2.358086109161377,
      "rewards/rejected": -3.197334051132202,
      "step": 1160
    },
    {
      "epoch": 0.61,
      "learning_rate": 2.1553228621291447e-07,
      "logits/chosen": -2.2347631454467773,
      "logits/rejected": -2.108426332473755,
      "logps/chosen": -282.07763671875,
      "logps/rejected": -275.0281982421875,
      "loss": 0.4327,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.1532267332077026,
      "rewards/margins": 2.362797498703003,
      "rewards/rejected": -3.516024112701416,
      "step": 1170
    },
    {
      "epoch": 0.62,
      "learning_rate": 2.1262361838278068e-07,
      "logits/chosen": -2.277366876602173,
      "logits/rejected": -2.1091060638427734,
      "logps/chosen": -299.38275146484375,
      "logps/rejected": -269.4599609375,
      "loss": 0.2882,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.1344071626663208,
      "rewards/margins": 2.7073185443878174,
      "rewards/rejected": -3.8417255878448486,
      "step": 1180
    },
    {
      "epoch": 0.62,
      "learning_rate": 2.0971495055264688e-07,
      "logits/chosen": -2.2836697101593018,
      "logits/rejected": -2.1914501190185547,
      "logps/chosen": -287.3064880371094,
      "logps/rejected": -263.22406005859375,
      "loss": 0.316,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.27397674322128296,
      "rewards/margins": 2.519117593765259,
      "rewards/rejected": -2.7930939197540283,
      "step": 1190
    },
    {
      "epoch": 0.63,
      "learning_rate": 2.0680628272251307e-07,
      "logits/chosen": -2.2928688526153564,
      "logits/rejected": -2.176222085952759,
      "logps/chosen": -279.0005187988281,
      "logps/rejected": -303.7047424316406,
      "loss": 0.359,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.5041743516921997,
      "rewards/margins": 2.464326858520508,
      "rewards/rejected": -2.968501329421997,
      "step": 1200
    },
    {
      "epoch": 0.63,
      "learning_rate": 2.0389761489237927e-07,
      "logits/chosen": -2.3216004371643066,
      "logits/rejected": -2.187290906906128,
      "logps/chosen": -282.7243957519531,
      "logps/rejected": -283.76904296875,
      "loss": 0.2816,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.3544381260871887,
      "rewards/margins": 2.6914665699005127,
      "rewards/rejected": -3.0459046363830566,
      "step": 1210
    },
    {
      "epoch": 0.64,
      "learning_rate": 2.0098894706224548e-07,
      "logits/chosen": -2.3324356079101562,
      "logits/rejected": -2.1998660564422607,
      "logps/chosen": -239.81320190429688,
      "logps/rejected": -246.91592407226562,
      "loss": 0.3403,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.48790687322616577,
      "rewards/margins": 2.5268945693969727,
      "rewards/rejected": -3.0148017406463623,
      "step": 1220
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.980802792321117e-07,
      "logits/chosen": -2.2460789680480957,
      "logits/rejected": -2.1282896995544434,
      "logps/chosen": -331.67144775390625,
      "logps/rejected": -274.24932861328125,
      "loss": 0.3193,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.4981555938720703,
      "rewards/margins": 2.2620322704315186,
      "rewards/rejected": -2.760187864303589,
      "step": 1230
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.9517161140197787e-07,
      "logits/chosen": -2.4429068565368652,
      "logits/rejected": -2.3785741329193115,
      "logps/chosen": -296.1825256347656,
      "logps/rejected": -311.277099609375,
      "loss": 0.324,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.26397719979286194,
      "rewards/margins": 2.5085294246673584,
      "rewards/rejected": -2.7725067138671875,
      "step": 1240
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.9226294357184408e-07,
      "logits/chosen": -2.3132803440093994,
      "logits/rejected": -2.1668436527252197,
      "logps/chosen": -297.1078186035156,
      "logps/rejected": -279.4066162109375,
      "loss": 0.3619,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.23463638126850128,
      "rewards/margins": 2.710824489593506,
      "rewards/rejected": -2.9454610347747803,
      "step": 1250
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.8935427574171028e-07,
      "logits/chosen": -2.3645033836364746,
      "logits/rejected": -2.1661763191223145,
      "logps/chosen": -317.00616455078125,
      "logps/rejected": -259.47955322265625,
      "loss": 0.3045,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.00973432045429945,
      "rewards/margins": 2.873765468597412,
      "rewards/rejected": -2.8640313148498535,
      "step": 1260
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.864456079115765e-07,
      "logits/chosen": -2.3081579208374023,
      "logits/rejected": -2.2391886711120605,
      "logps/chosen": -259.70318603515625,
      "logps/rejected": -275.3518981933594,
      "loss": 0.3005,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.2516987919807434,
      "rewards/margins": 2.7073814868927,
      "rewards/rejected": -2.959080219268799,
      "step": 1270
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.835369400814427e-07,
      "logits/chosen": -2.3735592365264893,
      "logits/rejected": -2.216593027114868,
      "logps/chosen": -287.75701904296875,
      "logps/rejected": -258.850341796875,
      "loss": 0.2848,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.4840475916862488,
      "rewards/margins": 2.552049160003662,
      "rewards/rejected": -3.0360965728759766,
      "step": 1280
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.8062827225130888e-07,
      "logits/chosen": -2.294996738433838,
      "logits/rejected": -2.132660150527954,
      "logps/chosen": -298.65631103515625,
      "logps/rejected": -278.43450927734375,
      "loss": 0.3069,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.8069319725036621,
      "rewards/margins": 2.365262985229492,
      "rewards/rejected": -3.1721949577331543,
      "step": 1290
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.777196044211751e-07,
      "logits/chosen": -2.289973735809326,
      "logits/rejected": -2.19386625289917,
      "logps/chosen": -263.17120361328125,
      "logps/rejected": -262.7763977050781,
      "loss": 0.3533,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5058833360671997,
      "rewards/margins": 2.9816837310791016,
      "rewards/rejected": -3.487567186355591,
      "step": 1300
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.748109365910413e-07,
      "logits/chosen": -2.2870774269104004,
      "logits/rejected": -2.1734931468963623,
      "logps/chosen": -235.30648803710938,
      "logps/rejected": -267.2663269042969,
      "loss": 0.3568,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.08289147913455963,
      "rewards/margins": 2.5056369304656982,
      "rewards/rejected": -2.422745704650879,
      "step": 1310
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.719022687609075e-07,
      "logits/chosen": -2.383655071258545,
      "logits/rejected": -2.16994309425354,
      "logps/chosen": -342.08282470703125,
      "logps/rejected": -297.896728515625,
      "loss": 0.4754,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.2975102961063385,
      "rewards/margins": 2.276179075241089,
      "rewards/rejected": -2.5736892223358154,
      "step": 1320
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.6899360093077368e-07,
      "logits/chosen": -2.3111634254455566,
      "logits/rejected": -2.197882890701294,
      "logps/chosen": -274.7708740234375,
      "logps/rejected": -243.1353759765625,
      "loss": 0.4027,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.430492103099823,
      "rewards/margins": 2.385375499725342,
      "rewards/rejected": -2.8158674240112305,
      "step": 1330
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.660849331006399e-07,
      "logits/chosen": -2.374952793121338,
      "logits/rejected": -2.1876838207244873,
      "logps/chosen": -325.642578125,
      "logps/rejected": -267.1546630859375,
      "loss": 0.2825,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.023454055190086365,
      "rewards/margins": 3.0091123580932617,
      "rewards/rejected": -2.9856581687927246,
      "step": 1340
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.631762652705061e-07,
      "logits/chosen": -2.3101236820220947,
      "logits/rejected": -2.2971105575561523,
      "logps/chosen": -227.51797485351562,
      "logps/rejected": -258.02642822265625,
      "loss": 0.2928,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.2340981513261795,
      "rewards/margins": 2.7252297401428223,
      "rewards/rejected": -2.9593281745910645,
      "step": 1350
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.602675974403723e-07,
      "logits/chosen": -2.3820908069610596,
      "logits/rejected": -2.2365851402282715,
      "logps/chosen": -270.7075500488281,
      "logps/rejected": -252.9586944580078,
      "loss": 0.3034,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.36182206869125366,
      "rewards/margins": 3.0204334259033203,
      "rewards/rejected": -3.3822555541992188,
      "step": 1360
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.573589296102385e-07,
      "logits/chosen": -2.3668997287750244,
      "logits/rejected": -2.200810670852661,
      "logps/chosen": -286.9949645996094,
      "logps/rejected": -287.580078125,
      "loss": 0.3634,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.7097588777542114,
      "rewards/margins": 2.247542142868042,
      "rewards/rejected": -2.957301139831543,
      "step": 1370
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.544502617801047e-07,
      "logits/chosen": -2.3244481086730957,
      "logits/rejected": -2.2366535663604736,
      "logps/chosen": -292.60992431640625,
      "logps/rejected": -292.54779052734375,
      "loss": 0.3295,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.16679362952709198,
      "rewards/margins": 2.6909871101379395,
      "rewards/rejected": -2.857780694961548,
      "step": 1380
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.515415939499709e-07,
      "logits/chosen": -2.213742256164551,
      "logits/rejected": -2.143993616104126,
      "logps/chosen": -292.03692626953125,
      "logps/rejected": -270.4903259277344,
      "loss": 0.3432,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.5152915120124817,
      "rewards/margins": 2.64485764503479,
      "rewards/rejected": -3.160149097442627,
      "step": 1390
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.486329261198371e-07,
      "logits/chosen": -2.2537856101989746,
      "logits/rejected": -2.138474702835083,
      "logps/chosen": -267.24884033203125,
      "logps/rejected": -268.0120849609375,
      "loss": 0.274,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7926814556121826,
      "rewards/margins": 2.544640064239502,
      "rewards/rejected": -3.3373215198516846,
      "step": 1400
    },
    {
      "epoch": 0.74,
      "learning_rate": 1.4572425828970332e-07,
      "logits/chosen": -2.3915152549743652,
      "logits/rejected": -2.170942544937134,
      "logps/chosen": -314.9630126953125,
      "logps/rejected": -270.1300964355469,
      "loss": 0.3387,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.9004882574081421,
      "rewards/margins": 2.366654872894287,
      "rewards/rejected": -3.2671427726745605,
      "step": 1410
    },
    {
      "epoch": 0.74,
      "learning_rate": 1.428155904595695e-07,
      "logits/chosen": -2.2214369773864746,
      "logits/rejected": -2.0794403553009033,
      "logps/chosen": -300.19415283203125,
      "logps/rejected": -284.22613525390625,
      "loss": 0.3149,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.6136606931686401,
      "rewards/margins": 3.0374813079833984,
      "rewards/rejected": -3.65114164352417,
      "step": 1420
    },
    {
      "epoch": 0.75,
      "learning_rate": 1.399069226294357e-07,
      "logits/chosen": -2.1593339443206787,
      "logits/rejected": -1.9660701751708984,
      "logps/chosen": -250.0261993408203,
      "logps/rejected": -228.28903198242188,
      "loss": 0.2967,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.6077468395233154,
      "rewards/margins": 2.963073253631592,
      "rewards/rejected": -3.570819854736328,
      "step": 1430
    },
    {
      "epoch": 0.75,
      "learning_rate": 1.3699825479930191e-07,
      "logits/chosen": -2.282439708709717,
      "logits/rejected": -2.080564498901367,
      "logps/chosen": -331.7917785644531,
      "logps/rejected": -233.09375,
      "loss": 0.3336,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.7863529324531555,
      "rewards/margins": 2.6935837268829346,
      "rewards/rejected": -3.479936122894287,
      "step": 1440
    },
    {
      "epoch": 0.76,
      "learning_rate": 1.3408958696916812e-07,
      "logits/chosen": -2.2574408054351807,
      "logits/rejected": -2.13787841796875,
      "logps/chosen": -265.936767578125,
      "logps/rejected": -284.001953125,
      "loss": 0.2295,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.5051341652870178,
      "rewards/margins": 3.3120696544647217,
      "rewards/rejected": -3.817204236984253,
      "step": 1450
    },
    {
      "epoch": 0.76,
      "learning_rate": 1.311809191390343e-07,
      "logits/chosen": -2.2214787006378174,
      "logits/rejected": -2.0134081840515137,
      "logps/chosen": -292.0023498535156,
      "logps/rejected": -292.8784484863281,
      "loss": 0.406,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.8032021522521973,
      "rewards/margins": 3.317984104156494,
      "rewards/rejected": -4.121186256408691,
      "step": 1460
    },
    {
      "epoch": 0.77,
      "learning_rate": 1.282722513089005e-07,
      "logits/chosen": -2.3705811500549316,
      "logits/rejected": -2.2448134422302246,
      "logps/chosen": -289.64300537109375,
      "logps/rejected": -262.7651062011719,
      "loss": 0.3156,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.44253435730934143,
      "rewards/margins": 2.5564846992492676,
      "rewards/rejected": -2.999018907546997,
      "step": 1470
    },
    {
      "epoch": 0.77,
      "learning_rate": 1.2536358347876672e-07,
      "logits/chosen": -2.2889206409454346,
      "logits/rejected": -2.227792501449585,
      "logps/chosen": -267.8201599121094,
      "logps/rejected": -258.48626708984375,
      "loss": 0.295,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.5226227045059204,
      "rewards/margins": 2.639317035675049,
      "rewards/rejected": -3.1619396209716797,
      "step": 1480
    },
    {
      "epoch": 0.78,
      "learning_rate": 1.2245491564863293e-07,
      "logits/chosen": -2.357090473175049,
      "logits/rejected": -2.142202615737915,
      "logps/chosen": -330.6837158203125,
      "logps/rejected": -281.90185546875,
      "loss": 0.2878,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.4217701852321625,
      "rewards/margins": 2.895638942718506,
      "rewards/rejected": -3.317409038543701,
      "step": 1490
    },
    {
      "epoch": 0.78,
      "learning_rate": 1.195462478184991e-07,
      "logits/chosen": -2.3041255474090576,
      "logits/rejected": -2.1481804847717285,
      "logps/chosen": -262.95391845703125,
      "logps/rejected": -265.3773193359375,
      "loss": 0.3618,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.6990595459938049,
      "rewards/margins": 2.7094345092773438,
      "rewards/rejected": -3.408494234085083,
      "step": 1500
    },
    {
      "epoch": 0.78,
      "eval_logits/chosen": -2.328040599822998,
      "eval_logits/rejected": -2.1793718338012695,
      "eval_logps/chosen": -289.56976318359375,
      "eval_logps/rejected": -277.8134460449219,
      "eval_loss": 0.34519457817077637,
      "eval_rewards/accuracies": 0.875,
      "eval_rewards/chosen": -0.4962008595466614,
      "eval_rewards/margins": 2.787445068359375,
      "eval_rewards/rejected": -3.2836461067199707,
      "eval_runtime": 718.9118,
      "eval_samples_per_second": 2.782,
      "eval_steps_per_second": 0.088,
      "step": 1500
    },
    {
      "epoch": 0.79,
      "learning_rate": 1.1663757998836531e-07,
      "logits/chosen": -2.265958786010742,
      "logits/rejected": -2.0687148571014404,
      "logps/chosen": -284.8194580078125,
      "logps/rejected": -248.05648803710938,
      "loss": 0.2666,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.4019474387168884,
      "rewards/margins": 3.251687526702881,
      "rewards/rejected": -3.653635025024414,
      "step": 1510
    },
    {
      "epoch": 0.8,
      "learning_rate": 1.1372891215823152e-07,
      "logits/chosen": -2.3123621940612793,
      "logits/rejected": -2.2089877128601074,
      "logps/chosen": -271.91351318359375,
      "logps/rejected": -272.6903076171875,
      "loss": 0.3357,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.38390928506851196,
      "rewards/margins": 2.608208417892456,
      "rewards/rejected": -2.9921176433563232,
      "step": 1520
    },
    {
      "epoch": 0.8,
      "learning_rate": 1.1082024432809772e-07,
      "logits/chosen": -2.300546646118164,
      "logits/rejected": -2.1590487957000732,
      "logps/chosen": -271.7149963378906,
      "logps/rejected": -244.64279174804688,
      "loss": 0.3112,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3754982352256775,
      "rewards/margins": 2.3724427223205566,
      "rewards/rejected": -2.747941255569458,
      "step": 1530
    },
    {
      "epoch": 0.81,
      "learning_rate": 1.0791157649796392e-07,
      "logits/chosen": -2.3611299991607666,
      "logits/rejected": -2.319636106491089,
      "logps/chosen": -261.41375732421875,
      "logps/rejected": -291.7661437988281,
      "loss": 0.3872,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.07669395208358765,
      "rewards/margins": 2.4173684120178223,
      "rewards/rejected": -2.34067440032959,
      "step": 1540
    },
    {
      "epoch": 0.81,
      "learning_rate": 1.0500290866783013e-07,
      "logits/chosen": -2.212877035140991,
      "logits/rejected": -2.085379123687744,
      "logps/chosen": -261.783203125,
      "logps/rejected": -255.25588989257812,
      "loss": 0.2901,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.059961654245853424,
      "rewards/margins": 2.9282031059265137,
      "rewards/rejected": -2.9881649017333984,
      "step": 1550
    },
    {
      "epoch": 0.82,
      "learning_rate": 1.0209424083769633e-07,
      "logits/chosen": -2.261627674102783,
      "logits/rejected": -2.1716694831848145,
      "logps/chosen": -273.3644714355469,
      "logps/rejected": -261.2357177734375,
      "loss": 0.3538,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.17286448180675507,
      "rewards/margins": 2.786151885986328,
      "rewards/rejected": -2.9590163230895996,
      "step": 1560
    },
    {
      "epoch": 0.82,
      "learning_rate": 9.918557300756253e-08,
      "logits/chosen": -2.303070545196533,
      "logits/rejected": -2.106238842010498,
      "logps/chosen": -286.3017883300781,
      "logps/rejected": -230.15576171875,
      "loss": 0.3195,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.3066893517971039,
      "rewards/margins": 3.2129788398742676,
      "rewards/rejected": -3.5196681022644043,
      "step": 1570
    },
    {
      "epoch": 0.83,
      "learning_rate": 9.627690517742873e-08,
      "logits/chosen": -2.271367311477661,
      "logits/rejected": -2.1737093925476074,
      "logps/chosen": -298.4072570800781,
      "logps/rejected": -290.3675842285156,
      "loss": 0.288,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.25732770562171936,
      "rewards/margins": 3.0521702766418457,
      "rewards/rejected": -2.794842481613159,
      "step": 1580
    },
    {
      "epoch": 0.83,
      "learning_rate": 9.336823734729494e-08,
      "logits/chosen": -2.271235227584839,
      "logits/rejected": -2.1804494857788086,
      "logps/chosen": -297.44476318359375,
      "logps/rejected": -279.959228515625,
      "loss": 0.2927,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.32572048902511597,
      "rewards/margins": 2.5924339294433594,
      "rewards/rejected": -2.918154239654541,
      "step": 1590
    },
    {
      "epoch": 0.84,
      "learning_rate": 9.045956951716113e-08,
      "logits/chosen": -2.292823076248169,
      "logits/rejected": -2.1955997943878174,
      "logps/chosen": -247.9260711669922,
      "logps/rejected": -235.42709350585938,
      "loss": 0.3524,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.17039936780929565,
      "rewards/margins": 2.6811909675598145,
      "rewards/rejected": -2.851590156555176,
      "step": 1600
    },
    {
      "epoch": 0.84,
      "learning_rate": 8.755090168702734e-08,
      "logits/chosen": -2.2323901653289795,
      "logits/rejected": -2.106067657470703,
      "logps/chosen": -279.44683837890625,
      "logps/rejected": -286.6934814453125,
      "loss": 0.2906,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.4923517107963562,
      "rewards/margins": 3.07497239112854,
      "rewards/rejected": -3.567323684692383,
      "step": 1610
    },
    {
      "epoch": 0.85,
      "learning_rate": 8.464223385689353e-08,
      "logits/chosen": -2.305323362350464,
      "logits/rejected": -2.233548641204834,
      "logps/chosen": -288.0763244628906,
      "logps/rejected": -268.65594482421875,
      "loss": 0.3689,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.27551114559173584,
      "rewards/margins": 2.574982166290283,
      "rewards/rejected": -2.8504936695098877,
      "step": 1620
    },
    {
      "epoch": 0.85,
      "learning_rate": 8.173356602675974e-08,
      "logits/chosen": -2.312969923019409,
      "logits/rejected": -2.1368892192840576,
      "logps/chosen": -264.1705627441406,
      "logps/rejected": -252.60848999023438,
      "loss": 0.3067,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.0068013668060303,
      "rewards/margins": 2.1217563152313232,
      "rewards/rejected": -3.1285574436187744,
      "step": 1630
    },
    {
      "epoch": 0.86,
      "learning_rate": 7.882489819662593e-08,
      "logits/chosen": -2.2855401039123535,
      "logits/rejected": -2.19065523147583,
      "logps/chosen": -282.502197265625,
      "logps/rejected": -347.73345947265625,
      "loss": 0.2969,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4646262228488922,
      "rewards/margins": 2.8088560104370117,
      "rewards/rejected": -3.273482084274292,
      "step": 1640
    },
    {
      "epoch": 0.86,
      "learning_rate": 7.591623036649214e-08,
      "logits/chosen": -2.1352343559265137,
      "logits/rejected": -2.088813066482544,
      "logps/chosen": -287.72955322265625,
      "logps/rejected": -267.5281677246094,
      "loss": 0.4208,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.43632712960243225,
      "rewards/margins": 2.592315196990967,
      "rewards/rejected": -3.028642177581787,
      "step": 1650
    },
    {
      "epoch": 0.87,
      "learning_rate": 7.300756253635834e-08,
      "logits/chosen": -2.300978183746338,
      "logits/rejected": -2.151585578918457,
      "logps/chosen": -266.4535827636719,
      "logps/rejected": -282.6994323730469,
      "loss": 0.3317,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.1617407649755478,
      "rewards/margins": 2.6548030376434326,
      "rewards/rejected": -2.8165440559387207,
      "step": 1660
    },
    {
      "epoch": 0.87,
      "learning_rate": 7.009889470622454e-08,
      "logits/chosen": -2.3190159797668457,
      "logits/rejected": -2.1111197471618652,
      "logps/chosen": -304.418701171875,
      "logps/rejected": -261.68865966796875,
      "loss": 0.296,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.013362640514969826,
      "rewards/margins": 3.4856789112091064,
      "rewards/rejected": -3.4990413188934326,
      "step": 1670
    },
    {
      "epoch": 0.88,
      "learning_rate": 6.719022687609075e-08,
      "logits/chosen": -2.247602939605713,
      "logits/rejected": -2.0082693099975586,
      "logps/chosen": -326.18359375,
      "logps/rejected": -308.87628173828125,
      "loss": 0.2498,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.1832130253314972,
      "rewards/margins": 3.8249638080596924,
      "rewards/rejected": -4.008176326751709,
      "step": 1680
    },
    {
      "epoch": 0.88,
      "learning_rate": 6.428155904595695e-08,
      "logits/chosen": -2.1355159282684326,
      "logits/rejected": -2.1326820850372314,
      "logps/chosen": -290.07147216796875,
      "logps/rejected": -286.3025207519531,
      "loss": 0.2702,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.38392263650894165,
      "rewards/margins": 3.633469343185425,
      "rewards/rejected": -3.249546766281128,
      "step": 1690
    },
    {
      "epoch": 0.89,
      "learning_rate": 6.137289121582315e-08,
      "logits/chosen": -2.3396739959716797,
      "logits/rejected": -2.0457844734191895,
      "logps/chosen": -317.99639892578125,
      "logps/rejected": -273.2770690917969,
      "loss": 0.3243,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.09367147088050842,
      "rewards/margins": 3.0268261432647705,
      "rewards/rejected": -3.120497941970825,
      "step": 1700
    },
    {
      "epoch": 0.89,
      "learning_rate": 5.846422338568935e-08,
      "logits/chosen": -2.2484824657440186,
      "logits/rejected": -2.0729587078094482,
      "logps/chosen": -283.7063293457031,
      "logps/rejected": -268.8601379394531,
      "loss": 0.2804,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.1501619815826416,
      "rewards/margins": 2.7162556648254395,
      "rewards/rejected": -2.8664181232452393,
      "step": 1710
    },
    {
      "epoch": 0.9,
      "learning_rate": 5.555555555555555e-08,
      "logits/chosen": -2.256999969482422,
      "logits/rejected": -2.092594623565674,
      "logps/chosen": -333.51324462890625,
      "logps/rejected": -271.24493408203125,
      "loss": 0.3285,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.4324091374874115,
      "rewards/margins": 2.6822280883789062,
      "rewards/rejected": -3.1146373748779297,
      "step": 1720
    },
    {
      "epoch": 0.91,
      "learning_rate": 5.264688772542175e-08,
      "logits/chosen": -2.190694570541382,
      "logits/rejected": -2.0392961502075195,
      "logps/chosen": -283.0891418457031,
      "logps/rejected": -250.1522979736328,
      "loss": 0.2861,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.07667428255081177,
      "rewards/margins": 2.9937844276428223,
      "rewards/rejected": -3.0704586505889893,
      "step": 1730
    },
    {
      "epoch": 0.91,
      "learning_rate": 4.973821989528795e-08,
      "logits/chosen": -2.285017490386963,
      "logits/rejected": -2.125302314758301,
      "logps/chosen": -256.6737365722656,
      "logps/rejected": -242.7133026123047,
      "loss": 0.3628,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.20822696387767792,
      "rewards/margins": 3.1273458003997803,
      "rewards/rejected": -2.919118881225586,
      "step": 1740
    },
    {
      "epoch": 0.92,
      "learning_rate": 4.682955206515416e-08,
      "logits/chosen": -2.3101308345794678,
      "logits/rejected": -2.1478629112243652,
      "logps/chosen": -289.7849426269531,
      "logps/rejected": -312.31524658203125,
      "loss": 0.3225,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10851460695266724,
      "rewards/margins": 3.011849880218506,
      "rewards/rejected": -3.12036395072937,
      "step": 1750
    },
    {
      "epoch": 0.92,
      "learning_rate": 4.392088423502036e-08,
      "logits/chosen": -2.308870553970337,
      "logits/rejected": -2.1753602027893066,
      "logps/chosen": -251.39785766601562,
      "logps/rejected": -231.73434448242188,
      "loss": 0.3189,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.27802008390426636,
      "rewards/margins": 2.905123233795166,
      "rewards/rejected": -3.183143377304077,
      "step": 1760
    },
    {
      "epoch": 0.93,
      "learning_rate": 4.101221640488656e-08,
      "logits/chosen": -2.264986276626587,
      "logits/rejected": -2.116339921951294,
      "logps/chosen": -288.02484130859375,
      "logps/rejected": -278.58892822265625,
      "loss": 0.2915,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.2735270857810974,
      "rewards/margins": 2.9817147254943848,
      "rewards/rejected": -3.255242109298706,
      "step": 1770
    },
    {
      "epoch": 0.93,
      "learning_rate": 3.810354857475276e-08,
      "logits/chosen": -2.1339287757873535,
      "logits/rejected": -1.9782625436782837,
      "logps/chosen": -311.86077880859375,
      "logps/rejected": -266.83270263671875,
      "loss": 0.3428,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.10870571434497833,
      "rewards/margins": 3.3522801399230957,
      "rewards/rejected": -3.4609856605529785,
      "step": 1780
    },
    {
      "epoch": 0.94,
      "learning_rate": 3.519488074461896e-08,
      "logits/chosen": -2.223109245300293,
      "logits/rejected": -2.0661864280700684,
      "logps/chosen": -251.7618865966797,
      "logps/rejected": -270.4107971191406,
      "loss": 0.253,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.07158003002405167,
      "rewards/margins": 3.161891460418701,
      "rewards/rejected": -3.090311288833618,
      "step": 1790
    },
    {
      "epoch": 0.94,
      "learning_rate": 3.228621291448516e-08,
      "logits/chosen": -2.2715201377868652,
      "logits/rejected": -2.0951197147369385,
      "logps/chosen": -315.7049255371094,
      "logps/rejected": -259.9617919921875,
      "loss": 0.3807,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.14460697770118713,
      "rewards/margins": 2.736274480819702,
      "rewards/rejected": -2.8808817863464355,
      "step": 1800
    },
    {
      "epoch": 0.95,
      "learning_rate": 2.9377545084351366e-08,
      "logits/chosen": -2.212463855743408,
      "logits/rejected": -2.1107747554779053,
      "logps/chosen": -365.3102111816406,
      "logps/rejected": -310.04986572265625,
      "loss": 0.3168,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.15414926409721375,
      "rewards/margins": 3.325876235961914,
      "rewards/rejected": -3.171726942062378,
      "step": 1810
    },
    {
      "epoch": 0.95,
      "learning_rate": 2.6468877254217567e-08,
      "logits/chosen": -2.240906000137329,
      "logits/rejected": -2.081852436065674,
      "logps/chosen": -307.33056640625,
      "logps/rejected": -293.54486083984375,
      "loss": 0.3318,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.0472014881670475,
      "rewards/margins": 3.150693416595459,
      "rewards/rejected": -3.103492021560669,
      "step": 1820
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.3560209424083768e-08,
      "logits/chosen": -2.2496564388275146,
      "logits/rejected": -2.1368825435638428,
      "logps/chosen": -304.8343200683594,
      "logps/rejected": -285.279052734375,
      "loss": 0.3409,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.13887426257133484,
      "rewards/margins": 2.7014544010162354,
      "rewards/rejected": -2.8403282165527344,
      "step": 1830
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.065154159394997e-08,
      "logits/chosen": -2.22584867477417,
      "logits/rejected": -2.1327052116394043,
      "logps/chosen": -314.4709777832031,
      "logps/rejected": -261.6623229980469,
      "loss": 0.4897,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.23914465308189392,
      "rewards/margins": 2.23769474029541,
      "rewards/rejected": -2.476839542388916,
      "step": 1840
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.7742873763816174e-08,
      "logits/chosen": -2.220412015914917,
      "logits/rejected": -2.1215548515319824,
      "logps/chosen": -315.8754577636719,
      "logps/rejected": -295.4253845214844,
      "loss": 0.3629,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12752804160118103,
      "rewards/margins": 3.037806510925293,
      "rewards/rejected": -3.165334701538086,
      "step": 1850
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.4834205933682373e-08,
      "logits/chosen": -2.270265817642212,
      "logits/rejected": -2.0846986770629883,
      "logps/chosen": -285.07525634765625,
      "logps/rejected": -246.28506469726562,
      "loss": 0.4131,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.09410500526428223,
      "rewards/margins": 3.04309344291687,
      "rewards/rejected": -3.1371986865997314,
      "step": 1860
    },
    {
      "epoch": 0.98,
      "learning_rate": 1.1925538103548575e-08,
      "logits/chosen": -2.172769546508789,
      "logits/rejected": -1.9685335159301758,
      "logps/chosen": -270.36553955078125,
      "logps/rejected": -245.4918670654297,
      "loss": 0.3484,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11144036054611206,
      "rewards/margins": 2.7038679122924805,
      "rewards/rejected": -2.815308094024658,
      "step": 1870
    },
    {
      "epoch": 0.98,
      "learning_rate": 9.016870273414776e-09,
      "logits/chosen": -2.237025737762451,
      "logits/rejected": -2.0877583026885986,
      "logps/chosen": -329.714111328125,
      "logps/rejected": -306.50347900390625,
      "loss": 0.2283,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.1830584704875946,
      "rewards/margins": 3.397716999053955,
      "rewards/rejected": -3.214658737182617,
      "step": 1880
    },
    {
      "epoch": 0.99,
      "learning_rate": 6.1082024432809765e-09,
      "logits/chosen": -2.13305401802063,
      "logits/rejected": -2.096346378326416,
      "logps/chosen": -262.7850036621094,
      "logps/rejected": -269.12371826171875,
      "loss": 0.3068,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.39133384823799133,
      "rewards/margins": 2.5701076984405518,
      "rewards/rejected": -2.9614415168762207,
      "step": 1890
    },
    {
      "epoch": 0.99,
      "learning_rate": 3.1995346131471783e-09,
      "logits/chosen": -2.2220492362976074,
      "logits/rejected": -2.0965094566345215,
      "logps/chosen": -257.00225830078125,
      "logps/rejected": -267.1251525878906,
      "loss": 0.318,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.30209580063819885,
      "rewards/margins": 3.022385358810425,
      "rewards/rejected": -3.324481248855591,
      "step": 1900
    },
    {
      "epoch": 1.0,
      "learning_rate": 2.9086678301337986e-10,
      "logits/chosen": -2.2612786293029785,
      "logits/rejected": -2.0825133323669434,
      "logps/chosen": -295.4743957519531,
      "logps/rejected": -249.2105255126953,
      "loss": 0.3212,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.1671399027109146,
      "rewards/margins": 2.760793447494507,
      "rewards/rejected": -2.9279332160949707,
      "step": 1910
    },
    {
      "epoch": 1.0,
      "step": 1911,
      "total_flos": 0.0,
      "train_loss": 0.3594729615826684,
      "train_runtime": 33818.9349,
      "train_samples_per_second": 1.808,
      "train_steps_per_second": 0.057
    }
  ],
  "logging_steps": 10,
  "max_steps": 1911,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}