{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9993753903810119,
  "eval_steps": 200,
  "global_step": 800,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.006246096189881324,
      "grad_norm": 116.66677856445312,
      "kl": 0.4583333432674408,
      "learning_rate": 1.5625e-07,
      "logits/chosen": 763024214.0595745,
      "logits/rejected": 257919736.68571427,
      "logps/chosen": -2094.8425531914895,
      "logps/rejected": -1494.726530612245,
      "loss": 0.5016,
      "rewards/chosen": 1.0933048329454786,
      "rewards/margins": -85873591.85363396,
      "rewards/rejected": 85873592.94693878,
      "step": 5
    },
    {
      "epoch": 0.012492192379762648,
      "grad_norm": 110.63726806640625,
      "kl": 0.38749998807907104,
      "learning_rate": 3.125e-07,
      "logits/chosen": 921293980.2033899,
      "logits/rejected": 220243934.4262295,
      "logps/chosen": -2251.3898305084745,
      "logps/rejected": -1449.1803278688524,
      "loss": 0.4946,
      "rewards/chosen": 1.1281716298248807,
      "rewards/margins": -41765219.59313984,
      "rewards/rejected": 41765220.72131147,
      "step": 10
    },
    {
      "epoch": 0.018738288569643973,
      "grad_norm": 121.6174087524414,
      "kl": 1.6833332777023315,
      "learning_rate": 4.6875000000000006e-07,
      "logits/chosen": 587306067.5021459,
      "logits/rejected": 338677312.2591093,
      "logps/chosen": -2082.74678111588,
      "logps/rejected": -1634.7206477732793,
      "loss": 0.4687,
      "rewards/chosen": 2.346424020922747,
      "rewards/margins": -31779756.811470717,
      "rewards/rejected": 31779759.157894738,
      "step": 15
    },
    {
      "epoch": 0.024984384759525295,
      "grad_norm": 68.7260971069336,
      "kl": 4.829166889190674,
      "learning_rate": 6.25e-07,
      "logits/chosen": 647625703.424,
      "logits/rejected": 289995090.3652174,
      "logps/chosen": -1992.704,
      "logps/rejected": -1608.904347826087,
      "loss": 0.4112,
      "rewards/chosen": 2.184742919921875,
      "rewards/margins": -24643590.719604906,
      "rewards/rejected": 24643592.904347826,
      "step": 20
    },
    {
      "epoch": 0.03123048094940662,
      "grad_norm": 51.2025260925293,
      "kl": 9.495833396911621,
      "learning_rate": 7.8125e-07,
      "logits/chosen": 728003276.3408072,
      "logits/rejected": 248500271.81322956,
      "logps/chosen": -2006.2421524663678,
      "logps/rejected": -1371.8910505836575,
      "loss": 0.2966,
      "rewards/chosen": 3.2302421261911434,
      "rewards/margins": 6841244.132965862,
      "rewards/rejected": -6841240.902723735,
      "step": 25
    },
    {
      "epoch": 0.037476577139287946,
      "grad_norm": 27.76913070678711,
      "kl": 22.079166412353516,
      "learning_rate": 9.375000000000001e-07,
      "logits/chosen": 918946912.2735043,
      "logits/rejected": 535911848.58536583,
      "logps/chosen": -2157.948717948718,
      "logps/rejected": -1666.081300813008,
      "loss": 0.1937,
      "rewards/chosen": 5.411676912226229,
      "rewards/margins": -80127611.6289735,
      "rewards/rejected": 80127617.04065041,
      "step": 30
    },
    {
      "epoch": 0.04372267332916927,
      "grad_norm": 33.331947326660156,
      "kl": 38.80833435058594,
      "learning_rate": 1.0937500000000001e-06,
      "logits/chosen": 992420333.4939759,
      "logits/rejected": 541292180.5021645,
      "logps/chosen": -2068.9477911646586,
      "logps/rejected": -1451.3593073593074,
      "loss": 0.1621,
      "rewards/chosen": 7.359968193084839,
      "rewards/margins": -22775727.497174665,
      "rewards/rejected": 22775734.85714286,
      "step": 35
    },
    {
      "epoch": 0.04996876951905059,
      "grad_norm": 19.21103286743164,
      "kl": 66.71666717529297,
      "learning_rate": 1.25e-06,
      "logits/chosen": 1329413437.2392156,
      "logits/rejected": 860745746.2044444,
      "logps/chosen": -2012.3607843137254,
      "logps/rejected": -1481.9555555555555,
      "loss": 0.1239,
      "rewards/chosen": 10.802983302696079,
      "rewards/margins": -30159119.365905587,
      "rewards/rejected": 30159130.16888889,
      "step": 40
    },
    {
      "epoch": 0.056214865708931916,
      "grad_norm": 10.822747230529785,
      "kl": 82.54166412353516,
      "learning_rate": 1.40625e-06,
      "logits/chosen": 1361087192.9491525,
      "logits/rejected": 947972868.1967213,
      "logps/chosen": -1899.3898305084747,
      "logps/rejected": -1508.983606557377,
      "loss": 0.1131,
      "rewards/chosen": 13.423981230137711,
      "rewards/margins": -74581901.85470729,
      "rewards/rejected": 74581915.27868852,
      "step": 45
    },
    {
      "epoch": 0.06246096189881324,
      "grad_norm": 7.292627811431885,
      "kl": 118.73332977294922,
      "learning_rate": 1.5625e-06,
      "logits/chosen": 1479708155.697479,
      "logits/rejected": 1002230673.983471,
      "logps/chosen": -1897.546218487395,
      "logps/rejected": -1337.9173553719008,
      "loss": 0.11,
      "rewards/chosen": 17.611668690913866,
      "rewards/margins": -83405586.35527346,
      "rewards/rejected": 83405603.96694215,
      "step": 50
    },
    {
      "epoch": 0.06870705808869457,
      "grad_norm": 7.088067054748535,
      "kl": 150.61666870117188,
      "learning_rate": 1.71875e-06,
      "logits/chosen": 1500882604.119149,
      "logits/rejected": 1047514584.2938776,
      "logps/chosen": -1893.1744680851064,
      "logps/rejected": -1392.3265306122448,
      "loss": 0.1315,
      "rewards/chosen": 22.309776013962765,
      "rewards/margins": -67494124.82491787,
      "rewards/rejected": 67494147.13469388,
      "step": 55
    },
    {
      "epoch": 0.07495315427857589,
      "grad_norm": 10.061450958251953,
      "kl": 160.98333740234375,
      "learning_rate": 1.8750000000000003e-06,
      "logits/chosen": 1491436649.9310346,
      "logits/rejected": 1136318133.6774194,
      "logps/chosen": -1901.5172413793102,
      "logps/rejected": -1438.0645161290322,
      "loss": 0.1269,
      "rewards/chosen": 22.11644245016164,
      "rewards/margins": -68507179.94807367,
      "rewards/rejected": 68507202.06451613,
      "step": 60
    },
    {
      "epoch": 0.08119925046845722,
      "grad_norm": 5.119268417358398,
      "kl": 169.3000030517578,
      "learning_rate": 2.0312500000000002e-06,
      "logits/chosen": 1529824284.1834862,
      "logits/rejected": 1174020908.946565,
      "logps/chosen": -1950.0183486238532,
      "logps/rejected": -1506.6870229007634,
      "loss": 0.1184,
      "rewards/chosen": 21.517562446244266,
      "rewards/margins": -64286292.00915511,
      "rewards/rejected": 64286313.52671756,
      "step": 65
    },
    {
      "epoch": 0.08744534665833854,
      "grad_norm": 7.102128028869629,
      "kl": 189.86666870117188,
      "learning_rate": 2.1875000000000002e-06,
      "logits/chosen": 1418054483.2263374,
      "logits/rejected": 1115578679.0886075,
      "logps/chosen": -1872.625514403292,
      "logps/rejected": -1406.649789029536,
      "loss": 0.1345,
      "rewards/chosen": 25.48732076260288,
      "rewards/margins": -91184910.47892396,
      "rewards/rejected": 91184935.96624473,
      "step": 70
    },
    {
      "epoch": 0.09369144284821987,
      "grad_norm": 6.652032852172852,
      "kl": 205.64999389648438,
      "learning_rate": 2.3437500000000002e-06,
      "logits/chosen": 1280121794.313253,
      "logits/rejected": 1075884368.9004328,
      "logps/chosen": -1890.8273092369477,
      "logps/rejected": -1434.874458874459,
      "loss": 0.1308,
      "rewards/chosen": 25.247278175200805,
      "rewards/margins": -58559015.0297781,
      "rewards/rejected": 58559040.27705628,
      "step": 75
    },
    {
      "epoch": 0.09993753903810118,
      "grad_norm": 7.803961277008057,
      "kl": 215.3000030517578,
      "learning_rate": 2.5e-06,
      "logits/chosen": 1125117697.0622406,
      "logits/rejected": 1087781335.2970712,
      "logps/chosen": -1695.8672199170123,
      "logps/rejected": -1510.2928870292887,
      "loss": 0.1215,
      "rewards/chosen": 27.956099260892117,
      "rewards/margins": -64467297.36607647,
      "rewards/rejected": 64467325.322175734,
      "step": 80
    },
    {
      "epoch": 0.1061836352279825,
      "grad_norm": 2.8079729080200195,
      "kl": 220.39999389648438,
      "learning_rate": 2.65625e-06,
      "logits/chosen": 1052407833.2839506,
      "logits/rejected": 1024613604.9957806,
      "logps/chosen": -1662.0246913580247,
      "logps/rejected": -1444.3206751054852,
      "loss": 0.1301,
      "rewards/chosen": 28.417864261831276,
      "rewards/margins": -75106206.45133404,
      "rewards/rejected": 75106234.8691983,
      "step": 85
    },
    {
      "epoch": 0.11242973141786383,
      "grad_norm": 8.319477081298828,
      "kl": 240.06666564941406,
      "learning_rate": 2.8125e-06,
      "logits/chosen": 995378244.2666667,
      "logits/rejected": 974057198.9333333,
      "logps/chosen": -1697.0666666666666,
      "logps/rejected": -1485.6,
      "loss": 0.1139,
      "rewards/chosen": 30.144661458333335,
      "rewards/margins": -43094211.98867188,
      "rewards/rejected": 43094242.13333333,
      "step": 90
    },
    {
      "epoch": 0.11867582760774516,
      "grad_norm": 7.453002452850342,
      "kl": 252.8333282470703,
      "learning_rate": 2.96875e-06,
      "logits/chosen": 1268567244.8,
      "logits/rejected": 996391867.7333333,
      "logps/chosen": -1984.1333333333334,
      "logps/rejected": -1418.2666666666667,
      "loss": 0.1239,
      "rewards/chosen": 33.62305908203125,
      "rewards/margins": -149487650.6436076,
      "rewards/rejected": 149487684.26666668,
      "step": 95
    },
    {
      "epoch": 0.12492192379762648,
      "grad_norm": 2.9787914752960205,
      "kl": 253.6666717529297,
      "learning_rate": 3.125e-06,
      "logits/chosen": 1126902571.1686275,
      "logits/rejected": 1192673644.088889,
      "logps/chosen": -1734.650980392157,
      "logps/rejected": -1441.9911111111112,
      "loss": 0.1281,
      "rewards/chosen": 28.739571844362747,
      "rewards/margins": -46898356.61153927,
      "rewards/rejected": 46898385.351111114,
      "step": 100
    },
    {
      "epoch": 0.1311680199875078,
      "grad_norm": 5.351078987121582,
      "kl": 260.0333251953125,
      "learning_rate": 3.28125e-06,
      "logits/chosen": 1035580677.3556485,
      "logits/rejected": 1086481369.759336,
      "logps/chosen": -1673.2384937238494,
      "logps/rejected": -1407.734439834025,
      "loss": 0.0947,
      "rewards/chosen": 30.54708543083159,
      "rewards/margins": -57016977.784864776,
      "rewards/rejected": 57017008.33195021,
      "step": 105
    },
    {
      "epoch": 0.13741411617738913,
      "grad_norm": 2.6211190223693848,
      "kl": 257.4333190917969,
      "learning_rate": 3.4375e-06,
      "logits/chosen": 992500800.5765766,
      "logits/rejected": 1126154367.007752,
      "logps/chosen": -1670.1981981981983,
      "logps/rejected": -1532.8992248062016,
      "loss": 0.086,
      "rewards/chosen": 30.3542700415259,
      "rewards/margins": -29103875.38216407,
      "rewards/rejected": 29103905.73643411,
      "step": 110
    },
    {
      "epoch": 0.14366021236727045,
      "grad_norm": 6.189810752868652,
      "kl": 277.76666259765625,
      "learning_rate": 3.59375e-06,
      "logits/chosen": 1039450317.6569037,
      "logits/rejected": 1082791774.5394192,
      "logps/chosen": -1658.510460251046,
      "logps/rejected": -1438.3402489626556,
      "loss": 0.1093,
      "rewards/chosen": 35.08254609048117,
      "rewards/margins": -68647662.9755452,
      "rewards/rejected": 68647698.05809128,
      "step": 115
    },
    {
      "epoch": 0.14990630855715179,
      "grad_norm": 2.9020302295684814,
      "kl": 283.3666687011719,
      "learning_rate": 3.7500000000000005e-06,
      "logits/chosen": 1015666845.5384616,
      "logits/rejected": 921090759.9656652,
      "logps/chosen": -1648.582995951417,
      "logps/rejected": -1456.6866952789699,
      "loss": 0.1231,
      "rewards/chosen": 38.58084514170041,
      "rewards/margins": -49614536.938468166,
      "rewards/rejected": 49614575.519313306,
      "step": 120
    },
    {
      "epoch": 0.1561524047470331,
      "grad_norm": 3.2500624656677246,
      "kl": 274.9333190917969,
      "learning_rate": 3.90625e-06,
      "logits/chosen": 739230529.0847458,
      "logits/rejected": 916730460.3278688,
      "logps/chosen": -1657.7627118644068,
      "logps/rejected": -1521.1803278688524,
      "loss": 0.1015,
      "rewards/chosen": 39.418812897245765,
      "rewards/margins": -48657276.25331825,
      "rewards/rejected": 48657315.67213115,
      "step": 125
    },
    {
      "epoch": 0.16239850093691444,
      "grad_norm": 2.6585230827331543,
      "kl": 280.6000061035156,
      "learning_rate": 4.0625000000000005e-06,
      "logits/chosen": 573097664.2700422,
      "logits/rejected": 1011293297.7777778,
      "logps/chosen": -1583.5274261603377,
      "logps/rejected": -1520.0658436213992,
      "loss": 0.1055,
      "rewards/chosen": 37.8720703125,
      "rewards/margins": -6502969.469493474,
      "rewards/rejected": 6503007.341563786,
      "step": 130
    },
    {
      "epoch": 0.16864459712679575,
      "grad_norm": 4.475651741027832,
      "kl": 298.6333312988281,
      "learning_rate": 4.21875e-06,
      "logits/chosen": 636761140.2039216,
      "logits/rejected": 859953488.7822223,
      "logps/chosen": -1657.349019607843,
      "logps/rejected": -1411.9822222222222,
      "loss": 0.0988,
      "rewards/chosen": 39.78631663602941,
      "rewards/margins": -48714953.422572255,
      "rewards/rejected": 48714993.20888889,
      "step": 135
    },
    {
      "epoch": 0.1748906933166771,
      "grad_norm": 2.648059606552124,
      "kl": 290.9333190917969,
      "learning_rate": 4.3750000000000005e-06,
      "logits/chosen": 607103567.2635983,
      "logits/rejected": 967892210.1908714,
      "logps/chosen": -1529.5732217573222,
      "logps/rejected": -1455.9336099585062,
      "loss": 0.0941,
      "rewards/chosen": 39.2620211166318,
      "rewards/margins": -50780460.65499133,
      "rewards/rejected": 50780499.917012446,
      "step": 140
    },
    {
      "epoch": 0.1811367895065584,
      "grad_norm": 4.189403533935547,
      "kl": 297.6333312988281,
      "learning_rate": 4.53125e-06,
      "logits/chosen": 732500241.0666667,
      "logits/rejected": 973196492.8,
      "logps/chosen": -1670.4,
      "logps/rejected": -1503.0666666666666,
      "loss": 0.1089,
      "rewards/chosen": 35.4449462890625,
      "rewards/margins": -47891689.35505371,
      "rewards/rejected": 47891724.8,
      "step": 145
    },
    {
      "epoch": 0.18738288569643974,
      "grad_norm": 1.987473726272583,
      "kl": 320.6000061035156,
      "learning_rate": 4.6875000000000004e-06,
      "logits/chosen": 786999262.4262295,
      "logits/rejected": 786822994.440678,
      "logps/chosen": -1613.311475409836,
      "logps/rejected": -1424.677966101695,
      "loss": 0.1051,
      "rewards/chosen": 43.21544409579918,
      "rewards/margins": -41480495.90319997,
      "rewards/rejected": 41480539.118644066,
      "step": 150
    },
    {
      "epoch": 0.19362898188632105,
      "grad_norm": 2.4980077743530273,
      "kl": 303.5,
      "learning_rate": 4.84375e-06,
      "logits/chosen": 865481696.4229075,
      "logits/rejected": 753880553.7391304,
      "logps/chosen": -1596.9691629955946,
      "logps/rejected": -1416.7272727272727,
      "loss": 0.0884,
      "rewards/chosen": 35.61597647645925,
      "rewards/margins": -22194391.324734986,
      "rewards/rejected": 22194426.94071146,
      "step": 155
    },
    {
      "epoch": 0.19987507807620236,
      "grad_norm": 2.3918511867523193,
      "kl": 324.6000061035156,
      "learning_rate": 5e-06,
      "logits/chosen": 928669237.248,
      "logits/rejected": 849364796.1043478,
      "logps/chosen": -1578.88,
      "logps/rejected": -1521.6695652173912,
      "loss": 0.0844,
      "rewards/chosen": 37.32334375,
      "rewards/margins": -24807967.6853519,
      "rewards/rejected": 24808005.00869565,
      "step": 160
    },
    {
      "epoch": 0.2061211742660837,
      "grad_norm": 3.1913163661956787,
      "kl": 323.3666687011719,
      "learning_rate": 4.999851262500375e-06,
      "logits/chosen": 989165542.0759493,
      "logits/rejected": 743453329.3827161,
      "logps/chosen": -1582.3122362869199,
      "logps/rejected": -1438.6172839506173,
      "loss": 0.0912,
      "rewards/chosen": 43.50049034150844,
      "rewards/margins": -5548838.145600193,
      "rewards/rejected": 5548881.6460905345,
      "step": 165
    },
    {
      "epoch": 0.212367270455965,
      "grad_norm": 1.8797273635864258,
      "kl": 319.6000061035156,
      "learning_rate": 4.999405067699773e-06,
      "logits/chosen": 816467482.0338984,
      "logits/rejected": 570425344.0,
      "logps/chosen": -1562.7796610169491,
      "logps/rejected": -1319.4754098360656,
      "loss": 0.0781,
      "rewards/chosen": 46.163491459216104,
      "rewards/margins": -39313294.688967556,
      "rewards/rejected": 39313340.85245901,
      "step": 170
    },
    {
      "epoch": 0.21861336664584635,
      "grad_norm": 3.3823835849761963,
      "kl": 350.0,
      "learning_rate": 4.998661468690914e-06,
      "logits/chosen": 859967620.1290323,
      "logits/rejected": 695513229.2413793,
      "logps/chosen": -1611.8709677419354,
      "logps/rejected": -1512.8275862068965,
      "loss": 0.1091,
      "rewards/chosen": 46.6678466796875,
      "rewards/margins": -36436881.33215332,
      "rewards/rejected": 36436928.0,
      "step": 175
    },
    {
      "epoch": 0.22485946283572766,
      "grad_norm": 5.648082733154297,
      "kl": 328.3999938964844,
      "learning_rate": 4.997620553954645e-06,
      "logits/chosen": 589463686.8477366,
      "logits/rejected": 560731546.464135,
      "logps/chosen": -1461.7283950617284,
      "logps/rejected": -1366.6835443037974,
      "loss": 0.0899,
      "rewards/chosen": 48.1337890625,
      "rewards/margins": -25789452.355662413,
      "rewards/rejected": 25789500.489451475,
      "step": 180
    },
    {
      "epoch": 0.231105559025609,
      "grad_norm": 2.2540504932403564,
      "kl": 370.29998779296875,
      "learning_rate": 4.996282447349408e-06,
      "logits/chosen": 559837283.902439,
      "logits/rejected": 561749945.982906,
      "logps/chosen": -1595.4471544715448,
      "logps/rejected": -1394.940170940171,
      "loss": 0.0976,
      "rewards/chosen": 53.93244648755081,
      "rewards/margins": -43921896.700032145,
      "rewards/rejected": 43921950.63247863,
      "step": 185
    },
    {
      "epoch": 0.23735165521549031,
      "grad_norm": 2.868093490600586,
      "kl": 339.73333740234375,
      "learning_rate": 4.994647308096509e-06,
      "logits/chosen": 453137114.8215768,
      "logits/rejected": 583618097.2719666,
      "logps/chosen": -1479.6348547717841,
      "logps/rejected": -1402.3096234309623,
      "loss": 0.116,
      "rewards/chosen": 47.16783097769709,
      "rewards/margins": -11266927.359365676,
      "rewards/rejected": 11266974.527196653,
      "step": 190
    },
    {
      "epoch": 0.24359775140537165,
      "grad_norm": 2.977546215057373,
      "kl": 345.9666748046875,
      "learning_rate": 4.992715330761167e-06,
      "logits/chosen": 526250719.17948717,
      "logits/rejected": 581422604.4878049,
      "logps/chosen": -1470.5641025641025,
      "logps/rejected": -1387.0569105691056,
      "loss": 0.1131,
      "rewards/chosen": 51.28399188701923,
      "rewards/margins": -30845854.927390225,
      "rewards/rejected": 30845906.211382113,
      "step": 195
    },
    {
      "epoch": 0.24984384759525297,
      "grad_norm": 4.593957424163818,
      "kl": 341.76666259765625,
      "learning_rate": 4.990486745229364e-06,
      "logits/chosen": 636709686.7008547,
      "logits/rejected": 676817445.4634147,
      "logps/chosen": -1580.854700854701,
      "logps/rejected": -1433.4959349593496,
      "loss": 0.0944,
      "rewards/chosen": 44.716037326388886,
      "rewards/margins": -2637613.8205480394,
      "rewards/rejected": 2637658.536585366,
      "step": 200
    },
    {
      "epoch": 0.24984384759525297,
      "eval_kl": 358.7391357421875,
      "eval_logits/chosen": 631036890.3675048,
      "eval_logits/rejected": 673337517.769697,
      "eval_logps/chosen": -1513.725338491296,
      "eval_logps/rejected": -1423.1353535353535,
      "eval_loss": 0.0891943946480751,
      "eval_rewards/chosen": 50.3120164410058,
      "eval_rewards/margins": -21855587.15060982,
      "eval_rewards/rejected": 21855637.462626263,
      "eval_runtime": 640.4487,
      "eval_samples_per_second": 6.316,
      "eval_steps_per_second": 0.395,
      "step": 200
    },
    {
      "epoch": 0.2560899437851343,
      "grad_norm": 4.819016933441162,
      "kl": 377.70001220703125,
      "learning_rate": 4.987961816680493e-06,
      "logits/chosen": 683028876.3870968,
      "logits/rejected": 758753209.3793104,
      "logps/chosen": -1516.774193548387,
      "logps/rejected": -1466.7586206896551,
      "loss": 0.0996,
      "rewards/chosen": 56.12273185483871,
      "rewards/margins": -41749567.04968194,
      "rewards/rejected": 41749623.172413796,
      "step": 205
    },
    {
      "epoch": 0.2623360399750156,
      "grad_norm": 2.632314682006836,
      "kl": 348.29998779296875,
      "learning_rate": 4.985140845555799e-06,
      "logits/chosen": 697918600.8097166,
      "logits/rejected": 637678218.4377682,
      "logps/chosen": -1600.1943319838056,
      "logps/rejected": -1350.0429184549357,
      "loss": 0.1253,
      "rewards/chosen": 54.92004443952429,
      "rewards/margins": -39001740.650771014,
      "rewards/rejected": 39001795.57081545,
      "step": 210
    },
    {
      "epoch": 0.26858213616489696,
      "grad_norm": 2.870805263519287,
      "kl": 350.8333435058594,
      "learning_rate": 4.982024167522638e-06,
      "logits/chosen": 527803308.06557375,
      "logits/rejected": 660025274.5762712,
      "logps/chosen": -1443.016393442623,
      "logps/rejected": -1426.4406779661017,
      "loss": 0.1019,
      "rewards/chosen": 50.20684314164959,
      "rewards/margins": -13903121.047394147,
      "rewards/rejected": 13903171.254237289,
      "step": 215
    },
    {
      "epoch": 0.27482823235477827,
      "grad_norm": 7.444933891296387,
      "kl": 368.73333740234375,
      "learning_rate": 4.978612153434527e-06,
      "logits/chosen": 546643296.5245901,
      "logits/rejected": 507599646.37288135,
      "logps/chosen": -1576.2622950819673,
      "logps/rejected": -1438.1016949152543,
      "loss": 0.0991,
      "rewards/chosen": 47.69948290215164,
      "rewards/margins": -18468254.402212013,
      "rewards/rejected": 18468302.101694915,
      "step": 220
    },
    {
      "epoch": 0.2810743285446596,
      "grad_norm": 0.719657838344574,
      "kl": 369.5,
      "learning_rate": 4.97490520928702e-06,
      "logits/chosen": 393574952.3651452,
      "logits/rejected": 569064717.9246862,
      "logps/chosen": -1517.5435684647302,
      "logps/rejected": -1512.3012552301254,
      "loss": 0.108,
      "rewards/chosen": 46.48416023729253,
      "rewards/margins": -8972016.043036416,
      "rewards/rejected": 8972062.527196653,
      "step": 225
    },
    {
      "epoch": 0.2873204247345409,
      "grad_norm": 4.509027481079102,
      "kl": 349.29998779296875,
      "learning_rate": 4.970903776169403e-06,
      "logits/chosen": 381775766.974359,
      "logits/rejected": 486850426.796748,
      "logps/chosen": -1601.6410256410256,
      "logps/rejected": -1370.4065040650407,
      "loss": 0.0894,
      "rewards/chosen": 49.49356887686966,
      "rewards/margins": -17199426.018626243,
      "rewards/rejected": 17199475.51219512,
      "step": 230
    },
    {
      "epoch": 0.29356652092442226,
      "grad_norm": 3.1007473468780518,
      "kl": 356.6000061035156,
      "learning_rate": 4.966608330212198e-06,
      "logits/chosen": 246713048.94915253,
      "logits/rejected": 590331098.2295082,
      "logps/chosen": -1364.2033898305085,
      "logps/rejected": -1417.967213114754,
      "loss": 0.0745,
      "rewards/chosen": 53.196975966631356,
      "rewards/margins": -17711701.294827312,
      "rewards/rejected": 17711754.491803277,
      "step": 235
    },
    {
      "epoch": 0.29981261711430357,
      "grad_norm": 1.0693767070770264,
      "kl": 363.70001220703125,
      "learning_rate": 4.962019382530521e-06,
      "logits/chosen": 238992315.73333332,
      "logits/rejected": 727606886.4,
      "logps/chosen": -1513.1291666666666,
      "logps/rejected": -1630.2666666666667,
      "loss": 0.1331,
      "rewards/chosen": 52.389933268229164,
      "rewards/margins": -38082861.74340007,
      "rewards/rejected": 38082914.13333333,
      "step": 240
    },
    {
      "epoch": 0.3060587133041849,
      "grad_norm": 1.2437098026275635,
      "kl": 341.76666259765625,
      "learning_rate": 4.957137479163253e-06,
      "logits/chosen": 186218359.46666667,
      "logits/rejected": 268291276.8,
      "logps/chosen": -1443.6,
      "logps/rejected": -1312.8666666666666,
      "loss": 0.0924,
      "rewards/chosen": 49.6269287109375,
      "rewards/margins": -20172466.639737956,
      "rewards/rejected": 20172516.266666666,
      "step": 245
    },
    {
      "epoch": 0.3123048094940662,
      "grad_norm": 1.5108606815338135,
      "kl": 359.4333190917969,
      "learning_rate": 4.9519632010080765e-06,
      "logits/chosen": 545861161.9672132,
      "logits/rejected": 418665072.8135593,
      "logps/chosen": -1472.9180327868853,
      "logps/rejected": -1367.7288135593221,
      "loss": 0.1153,
      "rewards/chosen": 42.937391937756146,
      "rewards/margins": -10470006.689726705,
      "rewards/rejected": 10470049.627118643,
      "step": 250
    },
    {
      "epoch": 0.3185509056839475,
      "grad_norm": 2.7949297428131104,
      "kl": 338.8333435058594,
      "learning_rate": 4.9464971637523465e-06,
      "logits/chosen": 841517825.1583711,
      "logits/rejected": 609372452.5714285,
      "logps/chosen": -1627.2217194570135,
      "logps/rejected": -1443.7065637065637,
      "loss": 0.0885,
      "rewards/chosen": 48.748851102941174,
      "rewards/margins": -26131600.98087863,
      "rewards/rejected": 26131649.72972973,
      "step": 255
    },
    {
      "epoch": 0.32479700187382887,
      "grad_norm": 1.7337244749069214,
      "kl": 370.8666687011719,
      "learning_rate": 4.9407400177998335e-06,
      "logits/chosen": 640891377.2510288,
      "logits/rejected": 656771374.4472574,
      "logps/chosen": -1558.2551440329219,
      "logps/rejected": -1530.1940928270042,
      "loss": 0.1031,
      "rewards/chosen": 46.06944846322016,
      "rewards/margins": -44654921.82928571,
      "rewards/rejected": 44654967.898734175,
      "step": 260
    },
    {
      "epoch": 0.3310430980637102,
      "grad_norm": 1.5853413343429565,
      "kl": 369.3333435058594,
      "learning_rate": 4.9346924481933345e-06,
      "logits/chosen": 461213069.5529412,
      "logits/rejected": 386304719.07555556,
      "logps/chosen": -1592.2196078431373,
      "logps/rejected": -1394.8444444444444,
      "loss": 0.1039,
      "rewards/chosen": 47.77142693014706,
      "rewards/margins": -23714219.81968418,
      "rewards/rejected": 23714267.591111112,
      "step": 265
    },
    {
      "epoch": 0.3372891942535915,
      "grad_norm": 5.782619953155518,
      "kl": 337.1666564941406,
      "learning_rate": 4.928355174533153e-06,
      "logits/chosen": 341149324.5023256,
      "logits/rejected": 302084853.3735849,
      "logps/chosen": -1583.739534883721,
      "logps/rejected": -1379.622641509434,
      "loss": 0.0806,
      "rewards/chosen": 50.5806640625,
      "rewards/margins": 121700.19575840211,
      "rewards/rejected": -121649.61509433962,
      "step": 270
    },
    {
      "epoch": 0.3435352904434728,
      "grad_norm": 2.3726727962493896,
      "kl": 349.76666259765625,
      "learning_rate": 4.9217289508914836e-06,
      "logits/chosen": 311504913.88646287,
      "logits/rejected": 560720794.0079681,
      "logps/chosen": -1507.8427947598254,
      "logps/rejected": -1589.9282868525897,
      "loss": 0.0565,
      "rewards/chosen": 49.57000972298035,
      "rewards/margins": -21957986.318436492,
      "rewards/rejected": 21958035.888446216,
      "step": 275
    },
    {
      "epoch": 0.3497813866333542,
      "grad_norm": 0.7196549773216248,
      "kl": 324.1666564941406,
      "learning_rate": 4.914814565722671e-06,
      "logits/chosen": 295960576.0,
      "logits/rejected": 450128081.63779527,
      "logps/chosen": -1468.7433628318583,
      "logps/rejected": -1489.5118110236222,
      "loss": 0.0737,
      "rewards/chosen": 47.86092194413717,
      "rewards/margins": -10037388.611519001,
      "rewards/rejected": 10037436.472440945,
      "step": 280
    },
    {
      "epoch": 0.3560274828232355,
      "grad_norm": 1.9283077716827393,
      "kl": 367.9333190917969,
      "learning_rate": 4.907612841769407e-06,
      "logits/chosen": 423632864.1245136,
      "logits/rejected": 392181530.40358746,
      "logps/chosen": -1474.739299610895,
      "logps/rejected": -1542.6726457399104,
      "loss": 0.091,
      "rewards/chosen": 51.97092731639105,
      "rewards/margins": -1841733.7151713383,
      "rewards/rejected": 1841785.6860986548,
      "step": 285
    },
    {
      "epoch": 0.3622735790131168,
      "grad_norm": 1.7306408882141113,
      "kl": 331.0666809082031,
      "learning_rate": 4.900124635964823e-06,
      "logits/chosen": 253768665.11392406,
      "logits/rejected": 425903091.3580247,
      "logps/chosen": -1416.5738396624472,
      "logps/rejected": -1596.4444444444443,
      "loss": 0.1082,
      "rewards/chosen": 47.69798259493671,
      "rewards/margins": -4647375.2485194625,
      "rewards/rejected": 4647422.946502058,
      "step": 290
    },
    {
      "epoch": 0.3685196752029981,
      "grad_norm": 1.1886324882507324,
      "kl": 312.8833312988281,
      "learning_rate": 4.8923508393305224e-06,
      "logits/chosen": 333225353.84615386,
      "logits/rejected": 267209547.29411766,
      "logps/chosen": -1454.076923076923,
      "logps/rejected": -1452.5294117647059,
      "loss": 0.0691,
      "rewards/chosen": 46.84036020132211,
      "rewards/margins": -18551674.571404506,
      "rewards/rejected": 18551721.411764707,
      "step": 295
    },
    {
      "epoch": 0.3747657713928795,
      "grad_norm": 1.4832735061645508,
      "kl": 345.0333251953125,
      "learning_rate": 4.884292376870567e-06,
      "logits/chosen": 251636303.26359832,
      "logits/rejected": 135267391.73443982,
      "logps/chosen": -1426.3430962343095,
      "logps/rejected": -1469.609958506224,
      "loss": 0.0914,
      "rewards/chosen": 49.49623676124477,
      "rewards/margins": -1139935.89795411,
      "rewards/rejected": 1139985.3941908714,
      "step": 300
    },
    {
      "epoch": 0.3810118675827608,
      "grad_norm": 2.8397765159606934,
      "kl": 313.3333435058594,
      "learning_rate": 4.875950207461403e-06,
      "logits/chosen": 39220267.02521008,
      "logits/rejected": -187643108.49586776,
      "logps/chosen": -1383.6638655462184,
      "logps/rejected": -1436.1652892561983,
      "loss": 0.0686,
      "rewards/chosen": 51.476660976890756,
      "rewards/margins": -7085320.820859684,
      "rewards/rejected": 7085372.297520661,
      "step": 305
    },
    {
      "epoch": 0.3872579637726421,
      "grad_norm": 1.789415717124939,
      "kl": 348.8333435058594,
      "learning_rate": 4.867325323737765e-06,
      "logits/chosen": 47862690.167330675,
      "logits/rejected": -399679166.04366815,
      "logps/chosen": -1544.5418326693227,
      "logps/rejected": -1370.480349344978,
      "loss": 0.1215,
      "rewards/chosen": 53.38380073456175,
      "rewards/margins": -2953413.0266796146,
      "rewards/rejected": 2953466.4104803493,
      "step": 310
    },
    {
      "epoch": 0.3935040599625234,
      "grad_norm": 2.056408643722534,
      "kl": 353.8999938964844,
      "learning_rate": 4.858418751974564e-06,
      "logits/chosen": 163274429.48031497,
      "logits/rejected": -516202133.5221239,
      "logps/chosen": -1566.1102362204724,
      "logps/rejected": -1494.4424778761063,
      "loss": 0.1019,
      "rewards/chosen": 53.50947342519685,
      "rewards/margins": -18813197.322384983,
      "rewards/rejected": 18813250.831858408,
      "step": 315
    },
    {
      "epoch": 0.3997501561524047,
      "grad_norm": 1.931511402130127,
      "kl": 337.4333190917969,
      "learning_rate": 4.849231551964771e-06,
      "logits/chosen": 35473859.25423729,
      "logits/rejected": -309613551.21311474,
      "logps/chosen": -1304.7457627118645,
      "logps/rejected": -1470.0983606557377,
      "loss": 0.0629,
      "rewards/chosen": 45.01601810778602,
      "rewards/margins": -11103810.197096646,
      "rewards/rejected": 11103855.213114753,
      "step": 320
    },
    {
      "epoch": 0.4059962523422861,
      "grad_norm": 1.5798239707946777,
      "kl": 345.73333740234375,
      "learning_rate": 4.839764816893315e-06,
      "logits/chosen": 356868378.48275864,
      "logits/rejected": -383262984.2580645,
      "logps/chosen": -1635.3103448275863,
      "logps/rejected": -1358.967741935484,
      "loss": 0.0973,
      "rewards/chosen": 44.951710668103445,
      "rewards/margins": -22694117.628934495,
      "rewards/rejected": 22694162.580645163,
      "step": 325
    },
    {
      "epoch": 0.4122423485321674,
      "grad_norm": 2.102696418762207,
      "kl": 361.3666687011719,
      "learning_rate": 4.830019673206997e-06,
      "logits/chosen": 375195714.0645161,
      "logits/rejected": -311598821.51724136,
      "logps/chosen": -1594.3225806451612,
      "logps/rejected": -1432.8275862068965,
      "loss": 0.0887,
      "rewards/chosen": 51.812842584425404,
      "rewards/margins": -5921475.911295347,
      "rewards/rejected": 5921527.724137931,
      "step": 330
    },
    {
      "epoch": 0.4184884447220487,
      "grad_norm": 2.166639566421509,
      "kl": 345.79998779296875,
      "learning_rate": 4.8199972804804615e-06,
      "logits/chosen": 103189217.94331984,
      "logits/rejected": -472300231.9656652,
      "logps/chosen": -1557.5060728744938,
      "logps/rejected": -1455.519313304721,
      "loss": 0.1055,
      "rewards/chosen": 48.8040193256579,
      "rewards/margins": -8893623.573663076,
      "rewards/rejected": 8893672.377682403,
      "step": 335
    },
    {
      "epoch": 0.42473454091193,
      "grad_norm": 2.084704875946045,
      "kl": 367.1666564941406,
      "learning_rate": 4.809698831278217e-06,
      "logits/chosen": 46631794.47154471,
      "logits/rejected": -579562294.7008547,
      "logps/chosen": -1487.3495934959349,
      "logps/rejected": -1438.4957264957266,
      "loss": 0.1104,
      "rewards/chosen": 54.046732088414636,
      "rewards/margins": 3384455.551005593,
      "rewards/rejected": -3384401.5042735045,
      "step": 340
    },
    {
      "epoch": 0.4309806371018114,
      "grad_norm": 1.7686203718185425,
      "kl": 361.9333190917969,
      "learning_rate": 4.799125551012731e-06,
      "logits/chosen": 187324373.1548117,
      "logits/rejected": -473956352.0,
      "logps/chosen": -1488.4686192468619,
      "logps/rejected": -1503.1369294605809,
      "loss": 0.0718,
      "rewards/chosen": 51.36217802039749,
      "rewards/margins": -14683061.268527374,
      "rewards/rejected": 14683112.630705394,
      "step": 345
    },
    {
      "epoch": 0.4372267332916927,
      "grad_norm": 1.8258565664291382,
      "kl": 373.1333312988281,
      "learning_rate": 4.788278697798619e-06,
      "logits/chosen": 254064045.41935483,
      "logits/rejected": -444822210.20689654,
      "logps/chosen": -1586.1935483870968,
      "logps/rejected": -1342.0689655172414,
      "loss": 0.1023,
      "rewards/chosen": 57.96908470892137,
      "rewards/margins": -6941889.617122187,
      "rewards/rejected": 6941947.586206896,
      "step": 350
    },
    {
      "epoch": 0.443472829481574,
      "grad_norm": 2.480158567428589,
      "kl": 339.4333190917969,
      "learning_rate": 4.77715956230294e-06,
      "logits/chosen": -27441811.525423728,
      "logits/rejected": -414471151.21311474,
      "logps/chosen": -1416.915254237288,
      "logps/rejected": -1494.950819672131,
      "loss": 0.1236,
      "rewards/chosen": 53.628765558792374,
      "rewards/margins": -2121243.682709851,
      "rewards/rejected": 2121297.3114754097,
      "step": 355
    },
    {
      "epoch": 0.4497189256714553,
      "grad_norm": 2.491856813430786,
      "kl": 345.29998779296875,
      "learning_rate": 4.765769467591626e-06,
      "logits/chosen": 31055325.866666667,
      "logits/rejected": -794077866.6666666,
      "logps/chosen": -1370.2666666666667,
      "logps/rejected": -1355.0666666666666,
      "loss": 0.0978,
      "rewards/chosen": 55.763850911458334,
      "rewards/margins": 1939260.5638509116,
      "rewards/rejected": -1939204.8,
      "step": 360
    },
    {
      "epoch": 0.45596502186133664,
      "grad_norm": 1.9237349033355713,
      "kl": 356.79998779296875,
      "learning_rate": 4.75410976897204e-06,
      "logits/chosen": 18777031.83122363,
      "logits/rejected": -744350875.9176955,
      "logps/chosen": -1626.464135021097,
      "logps/rejected": -1527.3086419753085,
      "loss": 0.0933,
      "rewards/chosen": 58.655549512130804,
      "rewards/margins": -8199585.525520447,
      "rewards/rejected": 8199644.181069959,
      "step": 365
    },
    {
      "epoch": 0.462211118051218,
      "grad_norm": 1.0969043970108032,
      "kl": 337.0666809082031,
      "learning_rate": 4.742181853831721e-06,
      "logits/chosen": -78409220.23140496,
      "logits/rejected": -775291981.4453782,
      "logps/chosen": -1493.0247933884298,
      "logps/rejected": -1540.5042016806722,
      "loss": 0.0874,
      "rewards/chosen": 53.18406185433884,
      "rewards/margins": 5562913.822717316,
      "rewards/rejected": -5562860.638655462,
      "step": 370
    },
    {
      "epoch": 0.4684572142410993,
      "grad_norm": 0.9217363595962524,
      "kl": 320.29998779296875,
      "learning_rate": 4.729987141473286e-06,
      "logits/chosen": -225435484.812749,
      "logits/rejected": -1214406691.7729259,
      "logps/chosen": -1473.1474103585658,
      "logps/rejected": -1494.2183406113538,
      "loss": 0.1034,
      "rewards/chosen": 45.022698269422314,
      "rewards/margins": -20387059.396515705,
      "rewards/rejected": 20387104.419213973,
      "step": 375
    },
    {
      "epoch": 0.47470331043098063,
      "grad_norm": 1.943109393119812,
      "kl": 350.0333251953125,
      "learning_rate": 4.717527082945555e-06,
      "logits/chosen": -90390528.0,
      "logits/rejected": -1083253906.2857144,
      "logps/chosen": -1475.875,
      "logps/rejected": -1701.142857142857,
      "loss": 0.1113,
      "rewards/chosen": 54.31856155395508,
      "rewards/margins": -1936764.824295589,
      "rewards/rejected": 1936819.142857143,
      "step": 380
    },
    {
      "epoch": 0.48094940662086194,
      "grad_norm": 2.9538779258728027,
      "kl": 337.5,
      "learning_rate": 4.704803160870888e-06,
      "logits/chosen": -34277903.650655024,
      "logits/rejected": -937418588.812749,
      "logps/chosen": -1378.8646288209607,
      "logps/rejected": -1622.9482071713148,
      "loss": 0.07,
      "rewards/chosen": 49.1605016716703,
      "rewards/margins": -2133147.2060321933,
      "rewards/rejected": 2133196.3665338648,
      "step": 385
    },
    {
      "epoch": 0.4871955028107433,
      "grad_norm": 2.6595630645751953,
      "kl": 388.5666809082031,
      "learning_rate": 4.69181688926877e-06,
      "logits/chosen": -68481698.44176707,
      "logits/rejected": -716681269.1948051,
      "logps/chosen": -1465.9598393574297,
      "logps/rejected": -1529.8354978354978,
      "loss": 0.1118,
      "rewards/chosen": 59.72445641942771,
      "rewards/margins": -2215760.7430760483,
      "rewards/rejected": 2215820.4675324676,
      "step": 390
    },
    {
      "epoch": 0.4934415990006246,
      "grad_norm": 1.581071376800537,
      "kl": 343.0,
      "learning_rate": 4.678569813375654e-06,
      "logits/chosen": -185697343.0900474,
      "logits/rejected": -681059857.1301116,
      "logps/chosen": -1491.4881516587677,
      "logps/rejected": -1469.5018587360594,
      "loss": 0.0735,
      "rewards/chosen": 55.95927595527251,
      "rewards/margins": 8371792.702770379,
      "rewards/rejected": -8371736.743494424,
      "step": 395
    },
    {
      "epoch": 0.49968769519050593,
      "grad_norm": 1.2356195449829102,
      "kl": 371.3666687011719,
      "learning_rate": 4.665063509461098e-06,
      "logits/chosen": -118915627.38983051,
      "logits/rejected": -537414538.4918033,
      "logps/chosen": -1546.7796610169491,
      "logps/rejected": -1465.5081967213114,
      "loss": 0.0858,
      "rewards/chosen": 58.09507829051907,
      "rewards/margins": 6046328.521307799,
      "rewards/rejected": -6046270.426229509,
      "step": 400
    },
    {
      "epoch": 0.49968769519050593,
      "eval_kl": 370.019775390625,
      "eval_logits/chosen": -210247537.3926499,
      "eval_logits/rejected": -512924984.37171715,
      "eval_logps/chosen": -1478.0309477756286,
      "eval_logps/rejected": -1445.3737373737374,
      "eval_loss": 0.08667240291833878,
      "eval_rewards/chosen": 53.16212826402321,
      "eval_rewards/margins": 7052260.483340385,
      "eval_rewards/rejected": -7052207.321212121,
      "eval_runtime": 640.6043,
      "eval_samples_per_second": 6.314,
      "eval_steps_per_second": 0.395,
      "step": 400
    },
    {
      "epoch": 0.5059337913803873,
      "grad_norm": 1.3852691650390625,
      "kl": 372.70001220703125,
      "learning_rate": 4.651299584640198e-06,
      "logits/chosen": -246257647.3495935,
      "logits/rejected": -504503969.9145299,
      "logps/chosen": -1464.1951219512196,
      "logps/rejected": -1501.8119658119658,
      "loss": 0.1236,
      "rewards/chosen": 42.6781035632622,
      "rewards/margins": 18530582.98579587,
      "rewards/rejected": -18530540.307692308,
      "step": 405
    },
    {
      "epoch": 0.5121798875702686,
      "grad_norm": 2.134881019592285,
      "kl": 359.29998779296875,
      "learning_rate": 4.637279676682367e-06,
      "logits/chosen": -342756045.2320675,
      "logits/rejected": -462322768.06584364,
      "logps/chosen": -1523.1729957805908,
      "logps/rejected": -1381.0041152263375,
      "loss": 0.0915,
      "rewards/chosen": 51.85403893064346,
      "rewards/margins": -1119816.1953437852,
      "rewards/rejected": 1119868.049382716,
      "step": 410
    },
    {
      "epoch": 0.5184259837601499,
      "grad_norm": 1.7998765707015991,
      "kl": 375.6666564941406,
      "learning_rate": 4.623005453816447e-06,
      "logits/chosen": -257030625.40239045,
      "logits/rejected": -576705352.6637554,
      "logps/chosen": -1588.398406374502,
      "logps/rejected": -1496.3842794759826,
      "loss": 0.1064,
      "rewards/chosen": 55.176921221364545,
      "rewards/margins": 15692810.460764015,
      "rewards/rejected": -15692755.283842795,
      "step": 415
    },
    {
      "epoch": 0.5246720799500312,
      "grad_norm": 1.7100578546524048,
      "kl": 369.79998779296875,
      "learning_rate": 4.608478614532215e-06,
      "logits/chosen": -132033194.66666667,
      "logits/rejected": -565794133.3333334,
      "logps/chosen": -1572.2666666666667,
      "logps/rejected": -1489.4666666666667,
      "loss": 0.0754,
      "rewards/chosen": 50.544681803385416,
      "rewards/margins": 18390476.144681804,
      "rewards/rejected": -18390425.6,
      "step": 420
    },
    {
      "epoch": 0.5309181761399125,
      "grad_norm": 2.343752145767212,
      "kl": 354.6000061035156,
      "learning_rate": 4.59370088737827e-06,
      "logits/chosen": -96648747.88571429,
      "logits/rejected": -456213107.4723404,
      "logps/chosen": -1476.7020408163266,
      "logps/rejected": -1570.4510638297872,
      "loss": 0.09,
      "rewards/chosen": 49.13263711734694,
      "rewards/margins": 19609119.702849884,
      "rewards/rejected": -19609070.570212767,
      "step": 425
    },
    {
      "epoch": 0.5371642723297939,
      "grad_norm": 1.4434489011764526,
      "kl": 314.0666809082031,
      "learning_rate": 4.578674030756364e-06,
      "logits/chosen": -89136869.51724137,
      "logits/rejected": -684162014.967742,
      "logps/chosen": -1368.1379310344828,
      "logps/rejected": -1453.4193548387098,
      "loss": 0.0911,
      "rewards/chosen": 39.44354037580819,
      "rewards/margins": 13124843.572572634,
      "rewards/rejected": -13124804.129032258,
      "step": 430
    },
    {
      "epoch": 0.5434103685196752,
      "grad_norm": 1.5906319618225098,
      "kl": 351.6000061035156,
      "learning_rate": 4.5633998327121595e-06,
      "logits/chosen": -210262649.3172691,
      "logits/rejected": -455542721.93939394,
      "logps/chosen": -1494.29718875502,
      "logps/rejected": -1558.5800865800866,
      "loss": 0.1078,
      "rewards/chosen": 57.3788827183735,
      "rewards/margins": 11366640.236025576,
      "rewards/rejected": -11366582.857142856,
      "step": 435
    },
    {
      "epoch": 0.5496564647095565,
      "grad_norm": 0.6508600115776062,
      "kl": 353.76666259765625,
      "learning_rate": 4.54788011072248e-06,
      "logits/chosen": -370147328.0,
      "logits/rejected": -258707238.66122448,
      "logps/chosen": -1327.1148936170214,
      "logps/rejected": -1479.7714285714285,
      "loss": 0.0935,
      "rewards/chosen": 56.83659408244681,
      "rewards/margins": -5150336.110344693,
      "rewards/rejected": 5150392.9469387755,
      "step": 440
    },
    {
      "epoch": 0.5559025608994379,
      "grad_norm": 2.842862606048584,
      "kl": 358.98333740234375,
      "learning_rate": 4.532116711479039e-06,
      "logits/chosen": -289068373.3333333,
      "logits/rejected": -292463082.1196581,
      "logps/chosen": -1573.7886178861788,
      "logps/rejected": -1477.8803418803418,
      "loss": 0.0997,
      "rewards/chosen": 52.84806910569106,
      "rewards/margins": 18087146.454906713,
      "rewards/rejected": -18087093.606837608,
      "step": 445
    },
    {
      "epoch": 0.5621486570893192,
      "grad_norm": 1.5763447284698486,
      "kl": 379.70001220703125,
      "learning_rate": 4.516111510668707e-06,
      "logits/chosen": -332198668.94941634,
      "logits/rejected": -401510565.30941707,
      "logps/chosen": -1566.7548638132296,
      "logps/rejected": -1400.7533632286995,
      "loss": 0.1167,
      "rewards/chosen": 60.13694674124513,
      "rewards/margins": 37355274.77371804,
      "rewards/rejected": -37355214.6367713,
      "step": 450
    },
    {
      "epoch": 0.5683947532792005,
      "grad_norm": 2.1485183238983154,
      "kl": 345.0,
      "learning_rate": 4.499866412750324e-06,
      "logits/chosen": -337919572.59130436,
      "logits/rejected": -610292203.52,
      "logps/chosen": -1506.0869565217392,
      "logps/rejected": -1416.576,
      "loss": 0.0855,
      "rewards/chosen": 42.14344429347826,
      "rewards/margins": 16386423.967444293,
      "rewards/rejected": -16386381.824,
      "step": 455
    },
    {
      "epoch": 0.5746408494690818,
      "grad_norm": 2.107536792755127,
      "kl": 378.3666687011719,
      "learning_rate": 4.4833833507280884e-06,
      "logits/chosen": -258783879.13754648,
      "logits/rejected": -399432912.6824645,
      "logps/chosen": -1535.1672862453531,
      "logps/rejected": -1555.8483412322275,
      "loss": 0.1178,
      "rewards/chosen": 57.77426158805762,
      "rewards/margins": 38428579.328763954,
      "rewards/rejected": -38428521.55450237,
      "step": 460
    },
    {
      "epoch": 0.5808869456589631,
      "grad_norm": 3.102811336517334,
      "kl": 390.3666687011719,
      "learning_rate": 4.466664285921543e-06,
      "logits/chosen": -51058047.02290076,
      "logits/rejected": -438891585.7614679,
      "logps/chosen": -1564.824427480916,
      "logps/rejected": -1454.4587155963302,
      "loss": 0.1278,
      "rewards/chosen": 65.5237058683206,
      "rewards/margins": 23682733.560403116,
      "rewards/rejected": -23682668.036697246,
      "step": 465
    },
    {
      "epoch": 0.5871330418488445,
      "grad_norm": 0.9413681626319885,
      "kl": 328.9333190917969,
      "learning_rate": 4.4497112077322045e-06,
      "logits/chosen": -152921083.60515022,
      "logits/rejected": -376898862.6396761,
      "logps/chosen": -1384.6523605150214,
      "logps/rejected": -1450.1052631578948,
      "loss": 0.0891,
      "rewards/chosen": 47.79664028969957,
      "rewards/margins": 6825927.63469697,
      "rewards/rejected": -6825879.83805668,
      "step": 470
    },
    {
      "epoch": 0.5933791380387258,
      "grad_norm": 2.5080554485321045,
      "kl": 356.1666564941406,
      "learning_rate": 4.432526133406843e-06,
      "logits/chosen": -447649817.70711297,
      "logits/rejected": -445022615.90041494,
      "logps/chosen": -1432.1004184100418,
      "logps/rejected": -1572.5145228215767,
      "loss": 0.0826,
      "rewards/chosen": 55.74282492154812,
      "rewards/margins": 9518347.128717037,
      "rewards/rejected": -9518291.385892116,
      "step": 475
    },
    {
      "epoch": 0.5996252342286071,
      "grad_norm": 1.6492382287979126,
      "kl": 355.1333312988281,
      "learning_rate": 4.415111107797445e-06,
      "logits/chosen": -344561655.84063745,
      "logits/rejected": -424582846.04366815,
      "logps/chosen": -1563.6653386454184,
      "logps/rejected": -1488.9082969432313,
      "loss": 0.1065,
      "rewards/chosen": 47.48057768924303,
      "rewards/margins": 6437849.401975069,
      "rewards/rejected": -6437801.92139738,
      "step": 480
    },
    {
      "epoch": 0.6058713304184884,
      "grad_norm": 3.074094533920288,
      "kl": 381.9333190917969,
      "learning_rate": 4.397468203117905e-06,
      "logits/chosen": -288593281.024,
      "logits/rejected": -461102177.9478261,
      "logps/chosen": -1617.92,
      "logps/rejected": -1586.3652173913044,
      "loss": 0.0895,
      "rewards/chosen": 55.55388671875,
      "rewards/margins": -3992440.5852437164,
      "rewards/rejected": 3992496.139130435,
      "step": 485
    },
    {
      "epoch": 0.6121174266083698,
      "grad_norm": 1.6913539171218872,
      "kl": 379.4666748046875,
      "learning_rate": 4.379599518697444e-06,
      "logits/chosen": -392446512.6692015,
      "logits/rejected": -512980774.9308756,
      "logps/chosen": -1402.1596958174905,
      "logps/rejected": -1331.0230414746543,
      "loss": 0.1057,
      "rewards/chosen": 50.04800751544677,
      "rewards/margins": 4576812.739251755,
      "rewards/rejected": -4576762.69124424,
      "step": 490
    },
    {
      "epoch": 0.6183635227982511,
      "grad_norm": 1.958149790763855,
      "kl": 344.79998779296875,
      "learning_rate": 4.3615071807308165e-06,
      "logits/chosen": -468381422.93333334,
      "logits/rejected": -774355899.7333333,
      "logps/chosen": -1462.5333333333333,
      "logps/rejected": -1496.6,
      "loss": 0.0993,
      "rewards/chosen": 55.822733561197914,
      "rewards/margins": 22640237.156066895,
      "rewards/rejected": -22640181.333333332,
      "step": 495
    },
    {
      "epoch": 0.6246096189881324,
      "grad_norm": 2.8394651412963867,
      "kl": 323.04998779296875,
      "learning_rate": 4.34319334202531e-06,
      "logits/chosen": -594912421.7095436,
      "logits/rejected": -977983582.2594142,
      "logps/chosen": -1415.966804979253,
      "logps/rejected": -1626.8451882845188,
      "loss": 0.0739,
      "rewards/chosen": 49.25540553682573,
      "rewards/margins": 6751152.18427583,
      "rewards/rejected": -6751102.928870293,
      "step": 500
    },
    {
      "epoch": 0.6308557151780138,
      "grad_norm": 1.808105230331421,
      "kl": 294.4166564941406,
      "learning_rate": 4.324660181744589e-06,
      "logits/chosen": -547166021.8181819,
      "logits/rejected": -1214759128.5286343,
      "logps/chosen": -1386.1185770750988,
      "logps/rejected": -1553.057268722467,
      "loss": 0.0818,
      "rewards/chosen": 58.81477349925889,
      "rewards/margins": 42604187.80155764,
      "rewards/rejected": -42604128.98678414,
      "step": 505
    },
    {
      "epoch": 0.637101811367895,
      "grad_norm": 2.6478588581085205,
      "kl": 282.5,
      "learning_rate": 4.305909905149389e-06,
      "logits/chosen": -299325648.97959185,
      "logits/rejected": -1448980323.1319149,
      "logps/chosen": -1492.6367346938775,
      "logps/rejected": -1649.8382978723405,
      "loss": 0.061,
      "rewards/chosen": 58.450175382653065,
      "rewards/margins": 16237909.684217935,
      "rewards/rejected": -16237851.234042553,
      "step": 510
    },
    {
      "epoch": 0.6433479075577764,
      "grad_norm": 2.0377795696258545,
      "kl": 277.3208312988281,
      "learning_rate": 4.2869447433351165e-06,
      "logits/chosen": -297782366.6554622,
      "logits/rejected": -1797935205.553719,
      "logps/chosen": -1581.7142857142858,
      "logps/rejected": -1638.3471074380166,
      "loss": 0.0794,
      "rewards/chosen": 51.58574054621849,
      "rewards/margins": 8687982.296484347,
      "rewards/rejected": -8687930.710743802,
      "step": 515
    },
    {
      "epoch": 0.6495940037476577,
      "grad_norm": 1.192832589149475,
      "kl": 161.88333129882812,
      "learning_rate": 4.267766952966369e-06,
      "logits/chosen": -465031805.15555555,
      "logits/rejected": -2354600024.345098,
      "logps/chosen": -1563.0222222222221,
      "logps/rejected": -1950.6196078431371,
      "loss": 0.0339,
      "rewards/chosen": 41.315043402777775,
      "rewards/margins": 4282941.393474775,
      "rewards/rejected": -4282900.0784313725,
      "step": 520
    },
    {
      "epoch": 0.655840099937539,
      "grad_norm": 2.186739206314087,
      "kl": 80.5999984741211,
      "learning_rate": 4.248378816008418e-06,
      "logits/chosen": -843879312.209607,
      "logits/rejected": -3130855766.6932273,
      "logps/chosen": -1512.3144104803494,
      "logps/rejected": -2192.191235059761,
      "loss": 0.0387,
      "rewards/chosen": 47.68359801446506,
      "rewards/margins": 26062714.13778128,
      "rewards/rejected": -26062666.454183266,
      "step": 525
    },
    {
      "epoch": 0.6620861961274204,
      "grad_norm": 1.2925801277160645,
      "kl": 111.38333129882812,
      "learning_rate": 4.228782639455674e-06,
      "logits/chosen": -785828956.2918454,
      "logits/rejected": -2260101559.449393,
      "logps/chosen": -1546.6437768240344,
      "logps/rejected": -2102.412955465587,
      "loss": 0.0262,
      "rewards/chosen": 46.72353641362661,
      "rewards/margins": 54497844.9421599,
      "rewards/rejected": -54497798.21862348,
      "step": 530
    },
    {
      "epoch": 0.6683322923173017,
      "grad_norm": 1.605604887008667,
      "kl": 66.55833435058594,
      "learning_rate": 4.2089807550571786e-06,
      "logits/chosen": -444578600.8739496,
      "logits/rejected": -2686018416.132231,
      "logps/chosen": -1527.5294117647059,
      "logps/rejected": -2166.7438016528927,
      "loss": 0.0224,
      "rewards/chosen": 52.38012900472689,
      "rewards/margins": 10841648.148724046,
      "rewards/rejected": -10841595.768595042,
      "step": 535
    },
    {
      "epoch": 0.674578388507183,
      "grad_norm": 0.6878402233123779,
      "kl": 81.24166870117188,
      "learning_rate": 4.188975519039151e-06,
      "logits/chosen": -626616434.688,
      "logits/rejected": -2936158688.8347826,
      "logps/chosen": -1452.16,
      "logps/rejected": -2022.9565217391305,
      "loss": 0.0303,
      "rewards/chosen": 50.23778515625,
      "rewards/margins": 3875481.1943068956,
      "rewards/rejected": -3875430.9565217393,
      "step": 540
    },
    {
      "epoch": 0.6808244846970644,
      "grad_norm": 3.325066566467285,
      "kl": 88.92082977294922,
      "learning_rate": 4.168769311824619e-06,
      "logits/chosen": -718940253.4771785,
      "logits/rejected": -2297285233.539749,
      "logps/chosen": -1545.0290456431535,
      "logps/rejected": -1931.6485355648535,
      "loss": 0.0296,
      "rewards/chosen": 48.45470532935685,
      "rewards/margins": 35838846.64717395,
      "rewards/rejected": -35838798.19246862,
      "step": 545
    },
    {
      "epoch": 0.6870705808869456,
      "grad_norm": 1.9588119983673096,
      "kl": 174.1750030517578,
      "learning_rate": 4.1483645377501726e-06,
      "logits/chosen": -284824458.7480916,
      "logits/rejected": -1198599327.706422,
      "logps/chosen": -1543.0839694656488,
      "logps/rejected": -1775.633027522936,
      "loss": 0.0587,
      "rewards/chosen": 39.873020783635496,
      "rewards/margins": -3450648.420557198,
      "rewards/rejected": 3450688.2935779816,
      "step": 550
    },
    {
      "epoch": 0.693316677076827,
      "grad_norm": 0.15676426887512207,
      "kl": 169.35833740234375,
      "learning_rate": 4.127763624779873e-06,
      "logits/chosen": -119682921.28138529,
      "logits/rejected": -928474042.0883534,
      "logps/chosen": -1460.5021645021645,
      "logps/rejected": -1611.9518072289156,
      "loss": 0.0322,
      "rewards/chosen": 50.89810775162338,
      "rewards/margins": -11360139.30269546,
      "rewards/rejected": 11360190.200803213,
      "step": 555
    },
    {
      "epoch": 0.6995627732667083,
      "grad_norm": 0.06072574853897095,
      "kl": 89.75833129882812,
      "learning_rate": 4.106969024216348e-06,
      "logits/chosen": -114480607.59493671,
      "logits/rejected": -2679107364.872428,
      "logps/chosen": -1609.3164556962026,
      "logps/rejected": -1946.732510288066,
      "loss": 0.0382,
      "rewards/chosen": 53.48231886207806,
      "rewards/margins": -20413582.46829842,
      "rewards/rejected": 20413635.950617284,
      "step": 560
    },
    {
      "epoch": 0.7058088694565896,
      "grad_norm": 1.8465758562088013,
      "kl": 24.049999237060547,
      "learning_rate": 4.085983210409114e-06,
      "logits/chosen": -160582556.6244726,
      "logits/rejected": -4038683239.2427983,
      "logps/chosen": -1587.9831223628692,
      "logps/rejected": -2597.135802469136,
      "loss": 0.0218,
      "rewards/chosen": 50.0565664556962,
      "rewards/margins": 1382332.5915458796,
      "rewards/rejected": -1382282.534979424,
      "step": 565
    },
    {
      "epoch": 0.712054965646471,
      "grad_norm": 0.00014929051394574344,
      "kl": 23.33333396911621,
      "learning_rate": 4.064808680460149e-06,
      "logits/chosen": -68750113.39130434,
      "logits/rejected": -3962644201.472,
      "logps/chosen": -1610.017391304348,
      "logps/rejected": -2563.84,
      "loss": 0.024,
      "rewards/chosen": 46.589041270380434,
      "rewards/margins": -8454041.60295873,
      "rewards/rejected": 8454088.192,
      "step": 570
    },
    {
      "epoch": 0.7183010618363522,
      "grad_norm": 0.03587708622217178,
      "kl": 22.733333587646484,
      "learning_rate": 4.043447953926763e-06,
      "logits/chosen": -123084586.29565218,
      "logits/rejected": -3554890743.808,
      "logps/chosen": -1611.9652173913043,
      "logps/rejected": -2491.136,
      "loss": 0.023,
      "rewards/chosen": 50.09292629076087,
      "rewards/margins": 7622541.740926291,
      "rewards/rejected": -7622491.648,
      "step": 575
    },
    {
      "epoch": 0.7245471580262336,
      "grad_norm": 0.06506123393774033,
      "kl": 25.579166412353516,
      "learning_rate": 4.021903572521802e-06,
      "logits/chosen": -45582472.53333333,
      "logits/rejected": -3599831313.0666666,
      "logps/chosen": -1660.9333333333334,
      "logps/rejected": -2363.4666666666667,
      "loss": 0.0177,
      "rewards/chosen": 51.65968424479167,
      "rewards/margins": -18986135.006982423,
      "rewards/rejected": 18986186.666666668,
      "step": 580
    },
    {
      "epoch": 0.730793254216115,
      "grad_norm": 0.004665783606469631,
      "kl": 13.354166984558105,
      "learning_rate": 4.000178099811203e-06,
      "logits/chosen": -82861664.73732719,
      "logits/rejected": -3341536610.311787,
      "logps/chosen": -1628.1658986175116,
      "logps/rejected": -2418.7376425855514,
      "loss": 0.0169,
      "rewards/chosen": 44.27093984014977,
      "rewards/margins": -4328426.603584874,
      "rewards/rejected": 4328470.874524714,
      "step": 585
    },
    {
      "epoch": 0.7370393504059962,
      "grad_norm": 0.7978938817977905,
      "kl": 21.72916603088379,
      "learning_rate": 3.978274120908957e-06,
      "logits/chosen": -186259499.94849786,
      "logits/rejected": -3371473252.534413,
      "logps/chosen": -1648.755364806867,
      "logps/rejected": -2399.093117408907,
      "loss": 0.0145,
      "rewards/chosen": 41.02078443535408,
      "rewards/margins": 4383058.478274315,
      "rewards/rejected": -4383017.457489879,
      "step": 590
    },
    {
      "epoch": 0.7432854465958776,
      "grad_norm": 0.05162600055336952,
      "kl": 55.141666412353516,
      "learning_rate": 3.956194242169506e-06,
      "logits/chosen": -199533438.92436975,
      "logits/rejected": -3286393170.512397,
      "logps/chosen": -1541.9159663865546,
      "logps/rejected": -2091.7685950413224,
      "loss": 0.0325,
      "rewards/chosen": 45.741264279149156,
      "rewards/margins": -4093033.7959258035,
      "rewards/rejected": 4093079.5371900825,
      "step": 595
    },
    {
      "epoch": 0.749531542785759,
      "grad_norm": 1.5164387226104736,
      "kl": 82.81666564941406,
      "learning_rate": 3.933941090877615e-06,
      "logits/chosen": -188560736.9531915,
      "logits/rejected": -2792379287.5102043,
      "logps/chosen": -1619.3361702127659,
      "logps/rejected": -2233.208163265306,
      "loss": 0.021,
      "rewards/chosen": 49.129488031914896,
      "rewards/margins": -826816.7643895191,
      "rewards/rejected": 826865.8938775511,
      "step": 600
    },
    {
      "epoch": 0.749531542785759,
      "eval_kl": 136.79397583007812,
      "eval_logits/chosen": -320818114.59961313,
      "eval_logits/rejected": -2315103287.8545456,
      "eval_logps/chosen": -1513.0328820116054,
      "eval_logps/rejected": -1956.0080808080809,
      "eval_loss": 0.02713991515338421,
      "eval_rewards/chosen": 50.27917976305609,
      "eval_rewards/margins": 11528678.772109056,
      "eval_rewards/rejected": -11528628.492929293,
      "eval_runtime": 640.8536,
      "eval_samples_per_second": 6.312,
      "eval_steps_per_second": 0.395,
      "step": 600
    },
    {
      "epoch": 0.7557776389756402,
      "grad_norm": 2.8158020973205566,
      "kl": 138.05416870117188,
      "learning_rate": 3.911517314935752e-06,
      "logits/chosen": -266606025.5319149,
      "logits/rejected": -2278140497.502041,
      "logps/chosen": -1721.9404255319148,
      "logps/rejected": -1823.3469387755101,
      "loss": 0.0452,
      "rewards/chosen": 49.07104803856383,
      "rewards/margins": 4880222.001660284,
      "rewards/rejected": -4880172.930612245,
      "step": 605
    },
    {
      "epoch": 0.7620237351655216,
      "grad_norm": 1.02464759349823,
      "kl": 175.93333435058594,
      "learning_rate": 3.888925582549006e-06,
      "logits/chosen": -262432676.79352227,
      "logits/rejected": -1933484137.476395,
      "logps/chosen": -1712.1943319838056,
      "logps/rejected": -1873.854077253219,
      "loss": 0.0411,
      "rewards/chosen": 50.334348114878544,
      "rewards/margins": 2964819.8450777284,
      "rewards/rejected": -2964769.5107296137,
      "step": 610
    },
    {
      "epoch": 0.7682698313554028,
      "grad_norm": 1.6535738706588745,
      "kl": 211.09165954589844,
      "learning_rate": 3.866168581907609e-06,
      "logits/chosen": -351476043.88932806,
      "logits/rejected": -1097745899.7004406,
      "logps/chosen": -1526.7667984189723,
      "logps/rejected": -1907.7004405286343,
      "loss": 0.0431,
      "rewards/chosen": 52.42443799407115,
      "rewards/margins": 19615485.164526097,
      "rewards/rejected": -19615432.740088105,
      "step": 615
    },
    {
      "epoch": 0.7745159275452842,
      "grad_norm": 1.2876919507980347,
      "kl": 227.76666259765625,
      "learning_rate": 3.8432490208670605e-06,
      "logits/chosen": -318065091.2542373,
      "logits/rejected": -671862179.6721312,
      "logps/chosen": -1523.2542372881355,
      "logps/rejected": -1821.377049180328,
      "loss": 0.0444,
      "rewards/chosen": 53.57854707362288,
      "rewards/margins": 4319505.644120844,
      "rewards/rejected": -4319452.065573771,
      "step": 620
    },
    {
      "epoch": 0.7807620237351656,
      "grad_norm": 2.2276229858398438,
      "kl": 182.30833435058594,
      "learning_rate": 3.82016962662592e-06,
      "logits/chosen": -279810331.1336405,
      "logits/rejected": -645603857.5209125,
      "logps/chosen": -1435.8709677419354,
      "logps/rejected": -1685.9011406844106,
      "loss": 0.0345,
      "rewards/chosen": 48.26090869815668,
      "rewards/margins": 16381825.051783223,
      "rewards/rejected": -16381776.790874524,
      "step": 625
    },
    {
      "epoch": 0.7870081199250468,
      "grad_norm": 2.2775721549987793,
      "kl": 211.5,
      "learning_rate": 3.796933145401304e-06,
      "logits/chosen": -10083805.866666667,
      "logits/rejected": -1062032725.3333334,
      "logps/chosen": -1574.8,
      "logps/rejected": -1685.7333333333333,
      "loss": 0.0462,
      "rewards/chosen": 56.797526041666664,
      "rewards/margins": -14378736.802473959,
      "rewards/rejected": 14378793.6,
      "step": 630
    },
    {
      "epoch": 0.7932542161149282,
      "grad_norm": 0.07378248870372772,
      "kl": 202.61666870117188,
      "learning_rate": 3.773542342102105e-06,
      "logits/chosen": -278961709.7886179,
      "logits/rejected": -1514538080.2735043,
      "logps/chosen": -1536.260162601626,
      "logps/rejected": -1685.8803418803418,
      "loss": 0.0386,
      "rewards/chosen": 51.39226054369919,
      "rewards/margins": 21470320.657217808,
      "rewards/rejected": -21470269.264957264,
      "step": 635
    },
    {
      "epoch": 0.7995003123048094,
      "grad_norm": 0.7576724290847778,
      "kl": 172.5416717529297,
      "learning_rate": 3.7500000000000005e-06,
      "logits/chosen": -356179968.0,
      "logits/rejected": -2250393892.571429,
      "logps/chosen": -1516.0,
      "logps/rejected": -1893.4285714285713,
      "loss": 0.051,
      "rewards/chosen": 53.19654846191406,
      "rewards/margins": 14372629.196548462,
      "rewards/rejected": -14372576.0,
      "step": 640
    },
    {
      "epoch": 0.8057464084946908,
      "grad_norm": 0.0,
      "kl": 102.68333435058594,
      "learning_rate": 3.7263089203982698e-06,
      "logits/chosen": 101189978.00913242,
      "logits/rejected": -2397663436.0153255,
      "logps/chosen": -1720.109589041096,
      "logps/rejected": -2050.206896551724,
      "loss": 0.0174,
      "rewards/chosen": 55.50405786244292,
      "rewards/margins": -22854268.971037924,
      "rewards/rejected": 22854324.475095786,
      "step": 645
    },
    {
      "epoch": 0.8119925046845722,
      "grad_norm": 1.2345607280731201,
      "kl": 101.40833282470703,
      "learning_rate": 3.7024719222984696e-06,
      "logits/chosen": 93082681.62770563,
      "logits/rejected": -2185956701.558233,
      "logps/chosen": -1410.3549783549784,
      "logps/rejected": -1973.0763052208836,
      "loss": 0.0334,
      "rewards/chosen": 41.46657281520563,
      "rewards/margins": 485957.8039222128,
      "rewards/rejected": -485916.3373493976,
      "step": 650
    },
    {
      "epoch": 0.8182386008744534,
      "grad_norm": 1.5449272394180298,
      "kl": 233.19166564941406,
      "learning_rate": 3.6784918420649952e-06,
      "logits/chosen": 125160275.22633745,
      "logits/rejected": -1470731812.7257383,
      "logps/chosen": -1487.8024691358025,
      "logps/rejected": -1760.4050632911392,
      "loss": 0.0414,
      "rewards/chosen": 51.10508696630659,
      "rewards/margins": -503803.34217041766,
      "rewards/rejected": 503854.44725738396,
      "step": 655
    },
    {
      "epoch": 0.8244846970643348,
      "grad_norm": 1.7102289199829102,
      "kl": 210.16250610351562,
      "learning_rate": 3.654371533087586e-06,
      "logits/chosen": -153904637.96015936,
      "logits/rejected": -1703949736.8034935,
      "logps/chosen": -1523.5059760956176,
      "logps/rejected": -1825.53711790393,
      "loss": 0.0511,
      "rewards/chosen": 51.47202595866534,
      "rewards/margins": -11561518.903519893,
      "rewards/rejected": 11561570.375545852,
      "step": 660
    },
    {
      "epoch": 0.8307307932542161,
      "grad_norm": 0.10204288363456726,
      "kl": 212.35833740234375,
      "learning_rate": 3.6301138654418e-06,
      "logits/chosen": -389819261.52895755,
      "logits/rejected": -1877226231.8914027,
      "logps/chosen": -1590.054054054054,
      "logps/rejected": -1905.6651583710407,
      "loss": 0.0442,
      "rewards/chosen": 55.06306180622587,
      "rewards/margins": 49894978.64677221,
      "rewards/rejected": -49894923.58371041,
      "step": 665
    },
    {
      "epoch": 0.8369768894440974,
      "grad_norm": 0.03783709183335304,
      "kl": 169.375,
      "learning_rate": 3.6057217255475034e-06,
      "logits/chosen": -503698552.22672063,
      "logits/rejected": -2175421672.9270387,
      "logps/chosen": -1393.748987854251,
      "logps/rejected": -1875.9141630901288,
      "loss": 0.0245,
      "rewards/chosen": 55.69806427125506,
      "rewards/margins": 31500282.170167275,
      "rewards/rejected": -31500226.472103003,
      "step": 670
    },
    {
      "epoch": 0.8432229856339788,
      "grad_norm": 2.866633892059326,
      "kl": 166.69166564941406,
      "learning_rate": 3.5811980158254156e-06,
      "logits/chosen": -492701266.1728395,
      "logits/rejected": -2529324589.367089,
      "logps/chosen": -1534.3539094650205,
      "logps/rejected": -1847.223628691983,
      "loss": 0.071,
      "rewards/chosen": 47.784866898148145,
      "rewards/margins": 21554439.818622172,
      "rewards/rejected": -21554392.033755273,
      "step": 675
    },
    {
      "epoch": 0.84946908182386,
      "grad_norm": 0.18893325328826904,
      "kl": 127.42500305175781,
      "learning_rate": 3.556545654351749e-06,
      "logits/chosen": -482326881.1034483,
      "logits/rejected": -2595327075.096774,
      "logps/chosen": -1636.4137931034484,
      "logps/rejected": -1988.774193548387,
      "loss": 0.0159,
      "rewards/chosen": 48.64041453394397,
      "rewards/margins": 6761048.89847905,
      "rewards/rejected": -6761000.258064516,
      "step": 680
    },
    {
      "epoch": 0.8557151780137414,
      "grad_norm": 0.03448840603232384,
      "kl": 138.22084045410156,
      "learning_rate": 3.531767574510987e-06,
      "logits/chosen": -554006615.5213675,
      "logits/rejected": -2420829508.6829267,
      "logps/chosen": -1462.837606837607,
      "logps/rejected": -2026.1463414634147,
      "loss": 0.0316,
      "rewards/chosen": 49.53289847088675,
      "rewards/margins": 21187200.622329365,
      "rewards/rejected": -21187151.089430895,
      "step": 685
    },
    {
      "epoch": 0.8619612742036228,
      "grad_norm": 1.088128685951233,
      "kl": 189.2083282470703,
      "learning_rate": 3.5068667246468437e-06,
      "logits/chosen": -323208633.2357724,
      "logits/rejected": -1934111875.2820513,
      "logps/chosen": -1403.5772357723577,
      "logps/rejected": -1723.3504273504273,
      "loss": 0.0325,
      "rewards/chosen": 49.48591526930894,
      "rewards/margins": -771720.4969907135,
      "rewards/rejected": 771769.9829059829,
      "step": 690
    },
    {
      "epoch": 0.868207370393504,
      "grad_norm": 2.381688117980957,
      "kl": 228.81666564941406,
      "learning_rate": 3.481846067711436e-06,
      "logits/chosen": -274805228.87966806,
      "logits/rejected": -1835806497.205021,
      "logps/chosen": -1443.3858921161825,
      "logps/rejected": -1687.0292887029289,
      "loss": 0.0513,
      "rewards/chosen": 49.824506450985474,
      "rewards/margins": 6338691.765929045,
      "rewards/rejected": -6338641.941422594,
      "step": 695
    },
    {
      "epoch": 0.8744534665833854,
      "grad_norm": 1.0362893342971802,
      "kl": 274.10833740234375,
      "learning_rate": 3.4567085809127247e-06,
      "logits/chosen": -263217944.77419356,
      "logits/rejected": -1258797409.1034484,
      "logps/chosen": -1521.1612903225807,
      "logps/rejected": -1691.5862068965516,
      "loss": 0.0462,
      "rewards/chosen": 50.86510049143145,
      "rewards/margins": 8044617.4857901465,
      "rewards/rejected": -8044566.620689655,
      "step": 700
    },
    {
      "epoch": 0.8806995627732667,
      "grad_norm": 1.926090121269226,
      "kl": 289.6333312988281,
      "learning_rate": 3.4314572553602577e-06,
      "logits/chosen": -373032056.73362446,
      "logits/rejected": -826461702.1195219,
      "logps/chosen": -1489.1877729257642,
      "logps/rejected": -1553.2749003984063,
      "loss": 0.0693,
      "rewards/chosen": 48.6717385371179,
      "rewards/margins": 10300293.325124992,
      "rewards/rejected": -10300244.653386455,
      "step": 705
    },
    {
      "epoch": 0.886945658963148,
      "grad_norm": 2.2127394676208496,
      "kl": 309.98333740234375,
      "learning_rate": 3.406095095709254e-06,
      "logits/chosen": -449325429.11740893,
      "logits/rejected": -790185272.0343348,
      "logps/chosen": -1486.7692307692307,
      "logps/rejected": -1538.3347639484978,
      "loss": 0.0561,
      "rewards/chosen": 56.23212139423077,
      "rewards/margins": 13225402.978902511,
      "rewards/rejected": -13225346.746781116,
      "step": 710
    },
    {
      "epoch": 0.8931917551530294,
      "grad_norm": 1.1908198595046997,
      "kl": 275.14166259765625,
      "learning_rate": 3.3806251198030843e-06,
      "logits/chosen": -674069715.5702479,
      "logits/rejected": -1077795142.9915967,
      "logps/chosen": -1509.9504132231405,
      "logps/rejected": -1646.2521008403362,
      "loss": 0.0592,
      "rewards/chosen": 55.71608664772727,
      "rewards/margins": 1421531.8841538746,
      "rewards/rejected": -1421476.168067227,
      "step": 715
    },
    {
      "epoch": 0.8994378513429107,
      "grad_norm": 0.20443065464496613,
      "kl": 221.72500610351562,
      "learning_rate": 3.3550503583141726e-06,
      "logits/chosen": -591554614.3716815,
      "logits/rejected": -1264764299.0866141,
      "logps/chosen": -1439.2920353982302,
      "logps/rejected": -1669.9212598425197,
      "loss": 0.0458,
      "rewards/chosen": 52.98895101631637,
      "rewards/margins": 10349140.233045503,
      "rewards/rejected": -10349087.244094487,
      "step": 720
    },
    {
      "epoch": 0.905683947532792,
      "grad_norm": 1.3105542659759521,
      "kl": 201.68333435058594,
      "learning_rate": 3.3293738543833807e-06,
      "logits/chosen": -319022639.3277311,
      "logits/rejected": -1492478950.6115704,
      "logps/chosen": -1498.218487394958,
      "logps/rejected": -1772.4297520661157,
      "loss": 0.0446,
      "rewards/chosen": 50.74974149816177,
      "rewards/margins": 1107236.2704026552,
      "rewards/rejected": -1107185.520661157,
      "step": 725
    },
    {
      "epoch": 0.9119300437226733,
      "grad_norm": 0.8841028213500977,
      "kl": 197.63333129882812,
      "learning_rate": 3.303598663257904e-06,
      "logits/chosen": -220504861.45132744,
      "logits/rejected": -1356576622.8661418,
      "logps/chosen": -1498.3362831858408,
      "logps/rejected": -1776.755905511811,
      "loss": 0.0464,
      "rewards/chosen": 44.9149915306969,
      "rewards/margins": -5135916.785795871,
      "rewards/rejected": 5135961.700787402,
      "step": 730
    },
    {
      "epoch": 0.9181761399125546,
      "grad_norm": 2.0225307941436768,
      "kl": 190.76666259765625,
      "learning_rate": 3.277727851927727e-06,
      "logits/chosen": -65942323.2,
      "logits/rejected": -1717986918.4,
      "logps/chosen": -1528.3333333333333,
      "logps/rejected": -1754.1333333333334,
      "loss": 0.0349,
      "rewards/chosen": 54.71465250651042,
      "rewards/margins": -17492478.618680827,
      "rewards/rejected": 17492533.333333332,
      "step": 735
    },
    {
      "epoch": 0.924422236102436,
      "grad_norm": 2.063528060913086,
      "kl": 142.30833435058594,
      "learning_rate": 3.2517644987606827e-06,
      "logits/chosen": 66764193.18518519,
      "logits/rejected": -1825793241.2121212,
      "logps/chosen": -1486.888888888889,
      "logps/rejected": -1923.2727272727273,
      "loss": 0.0417,
      "rewards/chosen": 53.85865614149306,
      "rewards/margins": -6370393.171646888,
      "rewards/rejected": 6370447.03030303,
      "step": 740
    },
    {
      "epoch": 0.9306683322923173,
      "grad_norm": 1.3004655838012695,
      "kl": 142.44166564941406,
      "learning_rate": 3.225711693136156e-06,
      "logits/chosen": 70914651.79282868,
      "logits/rejected": -2056784113.467249,
      "logps/chosen": -1554.4860557768925,
      "logps/rejected": -1979.9475982532751,
      "loss": 0.0384,
      "rewards/chosen": 48.3206937873506,
      "rewards/margins": -11652987.836511454,
      "rewards/rejected": 11653036.15720524,
      "step": 745
    },
    {
      "epoch": 0.9369144284821986,
      "grad_norm": 1.4171018600463867,
      "kl": 130.48333740234375,
      "learning_rate": 3.199572535077481e-06,
      "logits/chosen": -12038372.532188842,
      "logits/rejected": -2139264849.8785424,
      "logps/chosen": -1477.5965665236051,
      "logps/rejected": -1908.987854251012,
      "loss": 0.0385,
      "rewards/chosen": 46.0073598444206,
      "rewards/margins": 5270583.853513691,
      "rewards/rejected": -5270537.846153846,
      "step": 750
    },
    {
      "epoch": 0.94316052467208,
      "grad_norm": 0.09198802709579468,
      "kl": 131.65834045410156,
      "learning_rate": 3.173350134883066e-06,
      "logits/chosen": -136213255.89427313,
      "logits/rejected": -1772151463.9683795,
      "logps/chosen": -1524.7224669603524,
      "logps/rejected": -1885.3438735177865,
      "loss": 0.0153,
      "rewards/chosen": 51.28870543089207,
      "rewards/margins": 13132515.351946538,
      "rewards/rejected": -13132464.063241107,
      "step": 755
    },
    {
      "epoch": 0.9494066208619613,
      "grad_norm": 1.3991014957427979,
      "kl": 134.39999389648438,
      "learning_rate": 3.147047612756302e-06,
      "logits/chosen": -176903705.25560537,
      "logits/rejected": -1983824190.7548637,
      "logps/chosen": -1627.1210762331839,
      "logps/rejected": -2024.964980544747,
      "loss": 0.0407,
      "rewards/chosen": 53.05085569646861,
      "rewards/margins": -14139337.953035355,
      "rewards/rejected": 14139391.00389105,
      "step": 760
    },
    {
      "epoch": 0.9556527170518426,
      "grad_norm": 1.079622507095337,
      "kl": 134.00833129882812,
      "learning_rate": 3.120668098434291e-06,
      "logits/chosen": -444668686.5691057,
      "logits/rejected": -2044526031.8632479,
      "logps/chosen": -1559.219512195122,
      "logps/rejected": -2039.3846153846155,
      "loss": 0.0349,
      "rewards/chosen": 46.08423447027439,
      "rewards/margins": 484669.26372164977,
      "rewards/rejected": -484623.1794871795,
      "step": 765
    },
    {
      "epoch": 0.9618988132417239,
      "grad_norm": 2.1247825622558594,
      "kl": 173.84165954589844,
      "learning_rate": 3.094214730815433e-06,
      "logits/chosen": -334601040.33472806,
      "logits/rejected": -1739400493.6763484,
      "logps/chosen": -1507.7489539748954,
      "logps/rejected": -1843.1203319502074,
      "loss": 0.0627,
      "rewards/chosen": 54.40384741108787,
      "rewards/margins": 15768593.225424174,
      "rewards/rejected": -15768538.821576763,
      "step": 770
    },
    {
      "epoch": 0.9681449094316052,
      "grad_norm": 1.7619072198867798,
      "kl": 187.1666717529297,
      "learning_rate": 3.0676906575859335e-06,
      "logits/chosen": -414374765.71428573,
      "logits/rejected": -1798628478.9421487,
      "logps/chosen": -1502.3865546218487,
      "logps/rejected": -1810.5123966942149,
      "loss": 0.0286,
      "rewards/chosen": 41.90288126969538,
      "rewards/margins": 6427740.6797407735,
      "rewards/rejected": -6427698.776859504,
      "step": 775
    },
    {
      "epoch": 0.9743910056214866,
      "grad_norm": 1.2919367551803589,
      "kl": 171.06666564941406,
      "learning_rate": 3.0410990348452572e-06,
      "logits/chosen": -272265037.9130435,
      "logits/rejected": -1723221409.792,
      "logps/chosen": -1605.7043478260869,
      "logps/rejected": -1845.248,
      "loss": 0.044,
      "rewards/chosen": 47.08596297554348,
      "rewards/margins": 9132751.853962975,
      "rewards/rejected": -9132704.768,
      "step": 780
    },
    {
      "epoch": 0.9806371018113679,
      "grad_norm": 0.8556254506111145,
      "kl": 176.6458282470703,
      "learning_rate": 3.0144430267305874e-06,
      "logits/chosen": -508988111.07555556,
      "logits/rejected": -1627455745.0039215,
      "logps/chosen": -1427.6622222222222,
      "logps/rejected": -1869.0509803921568,
      "loss": 0.0208,
      "rewards/chosen": 53.00231336805555,
      "rewards/margins": 10830550.633685917,
      "rewards/rejected": -10830497.631372549,
      "step": 785
    },
    {
      "epoch": 0.9868831980012492,
      "grad_norm": 2.9301369190216064,
      "kl": 212.9499969482422,
      "learning_rate": 2.9877258050403214e-06,
      "logits/chosen": -519565405.8015267,
      "logits/rejected": -1667255079.9266055,
      "logps/chosen": -1437.1908396946565,
      "logps/rejected": -1811.9633027522937,
      "loss": 0.0375,
      "rewards/chosen": 57.16154281965649,
      "rewards/margins": 25719160.574386854,
      "rewards/rejected": -25719103.412844036,
      "step": 790
    },
    {
      "epoch": 0.9931292941911305,
      "grad_norm": 1.1394269466400146,
      "kl": 213.3249969482422,
      "learning_rate": 2.9609505488566585e-06,
      "logits/chosen": -600712120.5581396,
      "logits/rejected": -1835518118.054054,
      "logps/chosen": -1653.7054263565892,
      "logps/rejected": -1867.6756756756756,
      "loss": 0.0498,
      "rewards/chosen": 51.81218583454457,
      "rewards/margins": 37509377.07344709,
      "rewards/rejected": -37509325.26126126,
      "step": 795
    },
    {
      "epoch": 0.9993753903810119,
      "grad_norm": 0.3533008396625519,
      "kl": 138.0500030517578,
      "learning_rate": 2.9341204441673267e-06,
      "logits/chosen": -576359643.1091703,
      "logits/rejected": -1930248779.4741037,
      "logps/chosen": -1551.3711790393013,
      "logps/rejected": -1936.0637450199204,
      "loss": 0.0229,
      "rewards/chosen": 48.6797983760917,
      "rewards/margins": 5309753.859081244,
      "rewards/rejected": -5309705.179282868,
      "step": 800
    },
    {
      "epoch": 0.9993753903810119,
      "eval_kl": 164.8888397216797,
      "eval_logits/chosen": -603077229.926499,
      "eval_logits/rejected": -2010029103.579798,
      "eval_logps/chosen": -1488.5996131528047,
      "eval_logps/rejected": -1886.9171717171716,
      "eval_loss": 0.03083220310509205,
      "eval_rewards/chosen": 52.287400265957444,
      "eval_rewards/margins": 19438170.558107335,
      "eval_rewards/rejected": -19438118.27070707,
      "eval_runtime": 640.097,
      "eval_samples_per_second": 6.319,
      "eval_steps_per_second": 0.395,
      "step": 800
    }
  ],
  "logging_steps": 5,
  "max_steps": 1600,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 200,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}