{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 250,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.004,
      "grad_norm": 108.45250504320283,
      "learning_rate": 1.25e-07,
      "logits/chosen": -1.1796875,
      "logits/rejected": -1.1796875,
      "logps/chosen": -32.0,
      "logps/rejected": -35.25,
      "loss": 4.0839,
      "nll_loss": 3.421875,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.04150390625,
      "rewards/margins": 0.099609375,
      "rewards/rejected": -0.058349609375,
      "step": 1
    },
    {
      "epoch": 0.008,
      "grad_norm": 120.95987737750156,
      "learning_rate": 2.5e-07,
      "logits/chosen": -1.0703125,
      "logits/rejected": -1.0703125,
      "logps/chosen": -38.0,
      "logps/rejected": -35.25,
      "loss": 4.3121,
      "nll_loss": 3.984375,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.046142578125,
      "rewards/margins": 0.01171875,
      "rewards/rejected": 0.034423828125,
      "step": 2
    },
    {
      "epoch": 0.012,
      "grad_norm": 109.89014011391664,
      "learning_rate": 3.75e-07,
      "logits/chosen": -1.1875,
      "logits/rejected": -1.171875,
      "logps/chosen": -33.0,
      "logps/rejected": -37.0,
      "loss": 4.2,
      "nll_loss": 3.3125,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.05859375,
      "rewards/margins": 0.07421875,
      "rewards/rejected": -0.0155029296875,
      "step": 3
    },
    {
      "epoch": 0.016,
      "grad_norm": 105.87845131816027,
      "learning_rate": 5e-07,
      "logits/chosen": -1.234375,
      "logits/rejected": -1.203125,
      "logps/chosen": -31.25,
      "logps/rejected": -36.75,
      "loss": 4.2118,
      "nll_loss": 3.03125,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0595703125,
      "rewards/margins": 0.0390625,
      "rewards/rejected": 0.020263671875,
      "step": 4
    },
    {
      "epoch": 0.02,
      "grad_norm": 112.81877726787484,
      "learning_rate": 6.249999999999999e-07,
      "logits/chosen": -1.2265625,
      "logits/rejected": -1.2109375,
      "logps/chosen": -33.0,
      "logps/rejected": -34.25,
      "loss": 4.0945,
      "nll_loss": 3.453125,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.06298828125,
      "rewards/margins": -0.00994873046875,
      "rewards/rejected": 0.07275390625,
      "step": 5
    },
    {
      "epoch": 0.024,
      "grad_norm": 106.29693927531781,
      "learning_rate": 7.5e-07,
      "logits/chosen": -1.1796875,
      "logits/rejected": -1.1875,
      "logps/chosen": -33.0,
      "logps/rejected": -36.5,
      "loss": 4.1827,
      "nll_loss": 3.3125,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.08642578125,
      "rewards/margins": 0.037109375,
      "rewards/rejected": 0.04931640625,
      "step": 6
    },
    {
      "epoch": 0.028,
      "grad_norm": 114.94478205565038,
      "learning_rate": 8.75e-07,
      "logits/chosen": -1.1796875,
      "logits/rejected": -1.171875,
      "logps/chosen": -27.875,
      "logps/rejected": -33.5,
      "loss": 4.1499,
      "nll_loss": 2.84375,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0625,
      "rewards/margins": 0.01165771484375,
      "rewards/rejected": 0.051025390625,
      "step": 7
    },
    {
      "epoch": 0.032,
      "grad_norm": 97.37219322957407,
      "learning_rate": 1e-06,
      "logits/chosen": -1.171875,
      "logits/rejected": -1.171875,
      "logps/chosen": -32.5,
      "logps/rejected": -30.625,
      "loss": 4.0641,
      "nll_loss": 3.515625,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.1552734375,
      "rewards/margins": 0.030517578125,
      "rewards/rejected": 0.125,
      "step": 8
    },
    {
      "epoch": 0.036,
      "grad_norm": 108.05293741004135,
      "learning_rate": 9.999578688879084e-07,
      "logits/chosen": -1.15625,
      "logits/rejected": -1.1484375,
      "logps/chosen": -31.875,
      "logps/rejected": -37.25,
      "loss": 4.2044,
      "nll_loss": 3.234375,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.19140625,
      "rewards/margins": 0.055908203125,
      "rewards/rejected": 0.134765625,
      "step": 9
    },
    {
      "epoch": 0.04,
      "grad_norm": 89.47841349842803,
      "learning_rate": 9.998314826517563e-07,
      "logits/chosen": -1.1015625,
      "logits/rejected": -1.1171875,
      "logps/chosen": -27.75,
      "logps/rejected": -29.125,
      "loss": 3.8131,
      "nll_loss": 2.875,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.318359375,
      "rewards/margins": -0.049072265625,
      "rewards/rejected": 0.3671875,
      "step": 10
    },
    {
      "epoch": 0.044,
      "grad_norm": 82.2909196761966,
      "learning_rate": 9.99620862590714e-07,
      "logits/chosen": -1.1875,
      "logits/rejected": -1.1640625,
      "logps/chosen": -26.0,
      "logps/rejected": -30.625,
      "loss": 3.7873,
      "nll_loss": 2.640625,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.5390625,
      "rewards/margins": 0.028076171875,
      "rewards/rejected": 0.51171875,
      "step": 11
    },
    {
      "epoch": 0.048,
      "grad_norm": 77.5417011003063,
      "learning_rate": 9.993260441994114e-07,
      "logits/chosen": -1.1796875,
      "logits/rejected": -1.1328125,
      "logps/chosen": -26.375,
      "logps/rejected": -25.125,
      "loss": 3.6977,
      "nll_loss": 2.8125,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.515625,
      "rewards/margins": 0.02099609375,
      "rewards/rejected": 0.49609375,
      "step": 12
    },
    {
      "epoch": 0.052,
      "grad_norm": 77.36031138542103,
      "learning_rate": 9.989470771619553e-07,
      "logits/chosen": -1.09375,
      "logits/rejected": -1.109375,
      "logps/chosen": -31.375,
      "logps/rejected": -31.75,
      "loss": 3.84,
      "nll_loss": 3.171875,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.66015625,
      "rewards/margins": 0.0380859375,
      "rewards/rejected": 0.62109375,
      "step": 13
    },
    {
      "epoch": 0.056,
      "grad_norm": 38.474184849981015,
      "learning_rate": 9.984840253435568e-07,
      "logits/chosen": -1.1640625,
      "logits/rejected": -1.140625,
      "logps/chosen": -21.5,
      "logps/rejected": -23.125,
      "loss": 2.9964,
      "nll_loss": 2.15625,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 1.0625,
      "rewards/margins": 0.07958984375,
      "rewards/rejected": 0.984375,
      "step": 14
    },
    {
      "epoch": 0.06,
      "grad_norm": 37.90903240520294,
      "learning_rate": 9.979369667797674e-07,
      "logits/chosen": -1.0703125,
      "logits/rejected": -1.09375,
      "logps/chosen": -20.5,
      "logps/rejected": -25.25,
      "loss": 3.0721,
      "nll_loss": 2.0,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 1.078125,
      "rewards/margins": -0.060546875,
      "rewards/rejected": 1.140625,
      "step": 15
    },
    {
      "epoch": 0.064,
      "grad_norm": 35.90082325141723,
      "learning_rate": 9.973059936633306e-07,
      "logits/chosen": -1.078125,
      "logits/rejected": -1.1015625,
      "logps/chosen": -17.0,
      "logps/rejected": -25.25,
      "loss": 2.9806,
      "nll_loss": 1.71875,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.875,
      "rewards/margins": -0.0517578125,
      "rewards/rejected": 0.92578125,
      "step": 16
    },
    {
      "epoch": 0.068,
      "grad_norm": 34.39419041472641,
      "learning_rate": 9.965912123286424e-07,
      "logits/chosen": -0.99609375,
      "logits/rejected": -0.98046875,
      "logps/chosen": -21.875,
      "logps/rejected": -24.375,
      "loss": 2.9524,
      "nll_loss": 2.125,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 1.109375,
      "rewards/margins": 0.1552734375,
      "rewards/rejected": 0.95703125,
      "step": 17
    },
    {
      "epoch": 0.072,
      "grad_norm": 33.10304291271997,
      "learning_rate": 9.95792743233833e-07,
      "logits/chosen": -1.03125,
      "logits/rejected": -1.0078125,
      "logps/chosen": -18.75,
      "logps/rejected": -24.25,
      "loss": 2.8946,
      "nll_loss": 1.9765625,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 1.2265625,
      "rewards/margins": 0.0380859375,
      "rewards/rejected": 1.1875,
      "step": 18
    },
    {
      "epoch": 0.076,
      "grad_norm": 28.411495189507995,
      "learning_rate": 9.949107209404663e-07,
      "logits/chosen": -1.03125,
      "logits/rejected": -1.0234375,
      "logps/chosen": -19.5,
      "logps/rejected": -22.875,
      "loss": 2.7523,
      "nll_loss": 1.984375,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 1.5703125,
      "rewards/margins": 0.0096435546875,
      "rewards/rejected": 1.5625,
      "step": 19
    },
    {
      "epoch": 0.08,
      "grad_norm": 30.85467104218471,
      "learning_rate": 9.939452940908626e-07,
      "logits/chosen": -1.0703125,
      "logits/rejected": -1.046875,
      "logps/chosen": -20.875,
      "logps/rejected": -23.25,
      "loss": 2.8015,
      "nll_loss": 2.0625,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 1.6171875,
      "rewards/margins": 0.296875,
      "rewards/rejected": 1.3203125,
      "step": 20
    },
    {
      "epoch": 0.084,
      "grad_norm": 20.81104325101837,
      "learning_rate": 9.92896625383049e-07,
      "logits/chosen": -0.98828125,
      "logits/rejected": -0.9765625,
      "logps/chosen": -16.125,
      "logps/rejected": -21.0,
      "loss": 2.6257,
      "nll_loss": 1.640625,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 1.4609375,
      "rewards/margins": -0.09423828125,
      "rewards/rejected": 1.5546875,
      "step": 21
    },
    {
      "epoch": 0.088,
      "grad_norm": 16.555834400457428,
      "learning_rate": 9.917648915433411e-07,
      "logits/chosen": -0.95703125,
      "logits/rejected": -0.94140625,
      "logps/chosen": -15.8125,
      "logps/rejected": -16.875,
      "loss": 2.4551,
      "nll_loss": 1.53125,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 1.703125,
      "rewards/margins": 0.16796875,
      "rewards/rejected": 1.53125,
      "step": 22
    },
    {
      "epoch": 0.092,
      "grad_norm": 15.376428724494609,
      "learning_rate": 9.905502832965603e-07,
      "logits/chosen": -0.9296875,
      "logits/rejected": -0.91796875,
      "logps/chosen": -15.4375,
      "logps/rejected": -18.5,
      "loss": 2.4073,
      "nll_loss": 1.546875,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 1.6640625,
      "rewards/margins": -0.029052734375,
      "rewards/rejected": 1.6953125,
      "step": 23
    },
    {
      "epoch": 0.096,
      "grad_norm": 16.237442660961605,
      "learning_rate": 9.892530053338907e-07,
      "logits/chosen": -0.89453125,
      "logits/rejected": -0.8984375,
      "logps/chosen": -15.1875,
      "logps/rejected": -19.75,
      "loss": 2.3768,
      "nll_loss": 1.5546875,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 1.875,
      "rewards/margins": 0.08251953125,
      "rewards/rejected": 1.7890625,
      "step": 24
    },
    {
      "epoch": 0.1,
      "grad_norm": 15.758858425294349,
      "learning_rate": 9.87873276278386e-07,
      "logits/chosen": -0.8984375,
      "logits/rejected": -0.8828125,
      "logps/chosen": -14.5625,
      "logps/rejected": -16.875,
      "loss": 2.3852,
      "nll_loss": 1.4453125,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 2.125,
      "rewards/margins": 0.05029296875,
      "rewards/rejected": 2.078125,
      "step": 25
    },
    {
      "epoch": 0.104,
      "grad_norm": 16.347740539403798,
      "learning_rate": 9.864113286481236e-07,
      "logits/chosen": -0.9453125,
      "logits/rejected": -0.93359375,
      "logps/chosen": -14.125,
      "logps/rejected": -20.0,
      "loss": 2.317,
      "nll_loss": 1.46875,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 1.828125,
      "rewards/margins": -0.001220703125,
      "rewards/rejected": 1.828125,
      "step": 26
    },
    {
      "epoch": 0.108,
      "grad_norm": 15.099343265144629,
      "learning_rate": 9.84867408817022e-07,
      "logits/chosen": -0.95703125,
      "logits/rejected": -0.9375,
      "logps/chosen": -13.125,
      "logps/rejected": -16.625,
      "loss": 2.2059,
      "nll_loss": 1.328125,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 2.015625,
      "rewards/margins": 0.1494140625,
      "rewards/rejected": 1.859375,
      "step": 27
    },
    {
      "epoch": 0.112,
      "grad_norm": 15.421632558925108,
      "learning_rate": 9.832417769733185e-07,
      "logits/chosen": -0.94140625,
      "logits/rejected": -0.9375,
      "logps/chosen": -13.3125,
      "logps/rejected": -17.0,
      "loss": 2.2687,
      "nll_loss": 1.2578125,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 1.890625,
      "rewards/margins": 0.044921875,
      "rewards/rejected": 1.84375,
      "step": 28
    },
    {
      "epoch": 0.116,
      "grad_norm": 15.06033701313281,
      "learning_rate": 9.815347070757231e-07,
      "logits/chosen": -0.97265625,
      "logits/rejected": -0.9609375,
      "logps/chosen": -14.875,
      "logps/rejected": -17.875,
      "loss": 2.2442,
      "nll_loss": 1.375,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 2.109375,
      "rewards/margins": 0.3984375,
      "rewards/rejected": 1.71875,
      "step": 29
    },
    {
      "epoch": 0.12,
      "grad_norm": 13.532947619496891,
      "learning_rate": 9.797464868072486e-07,
      "logits/chosen": -0.91796875,
      "logits/rejected": -0.89453125,
      "logps/chosen": -13.0625,
      "logps/rejected": -16.75,
      "loss": 2.1401,
      "nll_loss": 1.3125,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 1.8203125,
      "rewards/margins": 0.18359375,
      "rewards/rejected": 1.6328125,
      "step": 30
    },
    {
      "epoch": 0.124,
      "grad_norm": 13.807093125286677,
      "learning_rate": 9.778774175267295e-07,
      "logits/chosen": -0.9375,
      "logits/rejected": -0.9296875,
      "logps/chosen": -11.75,
      "logps/rejected": -17.75,
      "loss": 2.1347,
      "nll_loss": 1.1953125,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 2.109375,
      "rewards/margins": 0.1552734375,
      "rewards/rejected": 1.9453125,
      "step": 31
    },
    {
      "epoch": 0.128,
      "grad_norm": 14.709867281760857,
      "learning_rate": 9.759278142180347e-07,
      "logits/chosen": -0.9765625,
      "logits/rejected": -0.94921875,
      "logps/chosen": -9.8125,
      "logps/rejected": -15.3125,
      "loss": 2.0944,
      "nll_loss": 0.98828125,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 1.96875,
      "rewards/margins": 0.11962890625,
      "rewards/rejected": 1.8515625,
      "step": 32
    },
    {
      "epoch": 0.132,
      "grad_norm": 12.980725519975202,
      "learning_rate": 9.73898005436987e-07,
      "logits/chosen": -0.95703125,
      "logits/rejected": -0.96484375,
      "logps/chosen": -11.0,
      "logps/rejected": -14.8125,
      "loss": 2.032,
      "nll_loss": 1.078125,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 1.9921875,
      "rewards/margins": 0.345703125,
      "rewards/rejected": 1.6484375,
      "step": 33
    },
    {
      "epoch": 0.136,
      "grad_norm": 12.423187407094442,
      "learning_rate": 9.71788333255991e-07,
      "logits/chosen": -0.97265625,
      "logits/rejected": -0.98046875,
      "logps/chosen": -12.0625,
      "logps/rejected": -15.125,
      "loss": 1.9855,
      "nll_loss": 1.2890625,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 2.125,
      "rewards/margins": 0.392578125,
      "rewards/rejected": 1.734375,
      "step": 34
    },
    {
      "epoch": 0.14,
      "grad_norm": 10.78340956295933,
      "learning_rate": 9.695991532063874e-07,
      "logits/chosen": -1.0078125,
      "logits/rejected": -0.97265625,
      "logps/chosen": -11.5,
      "logps/rejected": -14.5625,
      "loss": 1.9101,
      "nll_loss": 1.1484375,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.53125,
      "rewards/margins": 0.71875,
      "rewards/rejected": 1.8125,
      "step": 35
    },
    {
      "epoch": 0.144,
      "grad_norm": 10.539848066893843,
      "learning_rate": 9.673308342185365e-07,
      "logits/chosen": -0.9609375,
      "logits/rejected": -0.9609375,
      "logps/chosen": -11.6875,
      "logps/rejected": -16.625,
      "loss": 1.8997,
      "nll_loss": 1.1640625,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 2.21875,
      "rewards/margins": 0.224609375,
      "rewards/rejected": 1.984375,
      "step": 36
    },
    {
      "epoch": 0.148,
      "grad_norm": 10.7200500702218,
      "learning_rate": 9.649837585596444e-07,
      "logits/chosen": -0.921875,
      "logits/rejected": -0.9296875,
      "logps/chosen": -10.25,
      "logps/rejected": -16.125,
      "loss": 1.8621,
      "nll_loss": 0.94921875,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 2.171875,
      "rewards/margins": 0.345703125,
      "rewards/rejected": 1.828125,
      "step": 37
    },
    {
      "epoch": 0.152,
      "grad_norm": 9.54892533874284,
      "learning_rate": 9.625583217693418e-07,
      "logits/chosen": -0.93359375,
      "logits/rejected": -0.9453125,
      "logps/chosen": -11.125,
      "logps/rejected": -14.375,
      "loss": 1.7924,
      "nll_loss": 1.0703125,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 3.09375,
      "rewards/margins": 1.09375,
      "rewards/rejected": 1.9921875,
      "step": 38
    },
    {
      "epoch": 0.156,
      "grad_norm": 10.315046477409297,
      "learning_rate": 9.600549325930259e-07,
      "logits/chosen": -0.921875,
      "logits/rejected": -0.92578125,
      "logps/chosen": -10.375,
      "logps/rejected": -15.5,
      "loss": 1.8186,
      "nll_loss": 1.0546875,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 2.5625,
      "rewards/margins": 0.51171875,
      "rewards/rejected": 2.046875,
      "step": 39
    },
    {
      "epoch": 0.16,
      "grad_norm": 8.518386491338127,
      "learning_rate": 9.574740129129765e-07,
      "logits/chosen": -0.92578125,
      "logits/rejected": -0.90234375,
      "logps/chosen": -10.8125,
      "logps/rejected": -14.4375,
      "loss": 1.7032,
      "nll_loss": 1.0234375,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 2.015625,
      "rewards/margins": 0.07958984375,
      "rewards/rejected": 1.9375,
      "step": 40
    },
    {
      "epoch": 0.164,
      "grad_norm": 9.17761100715551,
      "learning_rate": 9.548159976772592e-07,
      "logits/chosen": -0.95703125,
      "logits/rejected": -0.93359375,
      "logps/chosen": -9.375,
      "logps/rejected": -14.375,
      "loss": 1.8061,
      "nll_loss": 0.93359375,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 2.4375,
      "rewards/margins": 0.357421875,
      "rewards/rejected": 2.09375,
      "step": 41
    },
    {
      "epoch": 0.168,
      "grad_norm": 9.6427558562046,
      "learning_rate": 9.520813348264251e-07,
      "logits/chosen": -0.91015625,
      "logits/rejected": -0.90625,
      "logps/chosen": -8.6875,
      "logps/rejected": -20.75,
      "loss": 1.6474,
      "nll_loss": 0.82421875,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 2.8125,
      "rewards/margins": 0.98828125,
      "rewards/rejected": 1.8203125,
      "step": 42
    },
    {
      "epoch": 0.172,
      "grad_norm": 10.106867319730664,
      "learning_rate": 9.492704852180226e-07,
      "logits/chosen": -0.88671875,
      "logits/rejected": -0.8984375,
      "logps/chosen": -8.0625,
      "logps/rejected": -17.625,
      "loss": 1.6427,
      "nll_loss": 0.734375,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 2.609375,
      "rewards/margins": 0.9296875,
      "rewards/rejected": 1.671875,
      "step": 43
    },
    {
      "epoch": 0.176,
      "grad_norm": 9.780716883236895,
      "learning_rate": 9.463839225489318e-07,
      "logits/chosen": -0.91015625,
      "logits/rejected": -0.91015625,
      "logps/chosen": -9.5,
      "logps/rejected": -15.375,
      "loss": 1.7142,
      "nll_loss": 0.93359375,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 2.625,
      "rewards/margins": 0.515625,
      "rewards/rejected": 2.109375,
      "step": 44
    },
    {
      "epoch": 0.18,
      "grad_norm": 9.324567213910779,
      "learning_rate": 9.434221332755338e-07,
      "logits/chosen": -0.90234375,
      "logits/rejected": -0.88671875,
      "logps/chosen": -8.125,
      "logps/rejected": -12.125,
      "loss": 1.6251,
      "nll_loss": 0.81640625,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 2.515625,
      "rewards/margins": 0.138671875,
      "rewards/rejected": 2.375,
      "step": 45
    },
    {
      "epoch": 0.184,
      "grad_norm": 9.236623014788846,
      "learning_rate": 9.403856165317321e-07,
      "logits/chosen": -0.91015625,
      "logits/rejected": -0.9140625,
      "logps/chosen": -8.875,
      "logps/rejected": -18.75,
      "loss": 1.542,
      "nll_loss": 0.87890625,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 2.515625,
      "rewards/margins": 0.6796875,
      "rewards/rejected": 1.828125,
      "step": 46
    },
    {
      "epoch": 0.188,
      "grad_norm": 9.078752548744747,
      "learning_rate": 9.372748840448361e-07,
      "logits/chosen": -0.8828125,
      "logits/rejected": -0.8828125,
      "logps/chosen": -6.3125,
      "logps/rejected": -19.25,
      "loss": 1.5166,
      "nll_loss": 0.62890625,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.375,
      "rewards/margins": 0.68359375,
      "rewards/rejected": 1.6875,
      "step": 47
    },
    {
      "epoch": 0.192,
      "grad_norm": 9.007283454701437,
      "learning_rate": 9.340904600493219e-07,
      "logits/chosen": -0.890625,
      "logits/rejected": -0.90625,
      "logps/chosen": -7.15625,
      "logps/rejected": -17.125,
      "loss": 1.5793,
      "nll_loss": 0.78125,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 2.421875,
      "rewards/margins": 0.55078125,
      "rewards/rejected": 1.8671875,
      "step": 48
    },
    {
      "epoch": 0.196,
      "grad_norm": 9.111837952613337,
      "learning_rate": 9.308328811984869e-07,
      "logits/chosen": -0.92578125,
      "logits/rejected": -0.90234375,
      "logps/chosen": -7.375,
      "logps/rejected": -19.25,
      "loss": 1.4635,
      "nll_loss": 0.6796875,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.4375,
      "rewards/margins": 0.90625,
      "rewards/rejected": 1.5390625,
      "step": 49
    },
    {
      "epoch": 0.2,
      "grad_norm": 8.886170629930112,
      "learning_rate": 9.275026964740101e-07,
      "logits/chosen": -0.93359375,
      "logits/rejected": -0.9296875,
      "logps/chosen": -7.25,
      "logps/rejected": -14.3125,
      "loss": 1.4143,
      "nll_loss": 0.6796875,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 2.65625,
      "rewards/margins": 0.474609375,
      "rewards/rejected": 2.171875,
      "step": 50
    },
    {
      "epoch": 0.204,
      "grad_norm": 8.951054512095832,
      "learning_rate": 9.241004670934347e-07,
      "logits/chosen": -0.9140625,
      "logits/rejected": -0.88671875,
      "logps/chosen": -6.46875,
      "logps/rejected": -18.625,
      "loss": 1.4843,
      "nll_loss": 0.59375,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.5,
      "rewards/margins": 0.78515625,
      "rewards/rejected": 1.71875,
      "step": 51
    },
    {
      "epoch": 0.208,
      "grad_norm": 8.428410886373113,
      "learning_rate": 9.206267664155906e-07,
      "logits/chosen": -0.9140625,
      "logits/rejected": -0.8828125,
      "logps/chosen": -8.5625,
      "logps/rejected": -17.0,
      "loss": 1.5279,
      "nll_loss": 0.875,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 2.59375,
      "rewards/margins": 0.46875,
      "rewards/rejected": 2.125,
      "step": 52
    },
    {
      "epoch": 0.212,
      "grad_norm": 13.359303723068443,
      "learning_rate": 9.170821798439683e-07,
      "logits/chosen": -0.921875,
      "logits/rejected": -0.921875,
      "logps/chosen": -7.21875,
      "logps/rejected": -19.75,
      "loss": 1.4738,
      "nll_loss": 0.7265625,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 2.25,
      "rewards/margins": 0.84375,
      "rewards/rejected": 1.4140625,
      "step": 53
    },
    {
      "epoch": 0.216,
      "grad_norm": 8.607311027896719,
      "learning_rate": 9.134673047280643e-07,
      "logits/chosen": -0.90234375,
      "logits/rejected": -0.890625,
      "logps/chosen": -7.3125,
      "logps/rejected": -12.75,
      "loss": 1.5091,
      "nll_loss": 0.6875,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.53125,
      "rewards/margins": 0.66796875,
      "rewards/rejected": 1.859375,
      "step": 54
    },
    {
      "epoch": 0.22,
      "grad_norm": 10.84727026791628,
      "learning_rate": 9.097827502627136e-07,
      "logits/chosen": -0.890625,
      "logits/rejected": -0.9140625,
      "logps/chosen": -5.40625,
      "logps/rejected": -15.125,
      "loss": 1.3565,
      "nll_loss": 0.50390625,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 2.90625,
      "rewards/margins": 0.72265625,
      "rewards/rejected": 2.1875,
      "step": 55
    },
    {
      "epoch": 0.224,
      "grad_norm": 10.663731973556606,
      "learning_rate": 9.06029137385425e-07,
      "logits/chosen": -0.90234375,
      "logits/rejected": -0.875,
      "logps/chosen": -8.0625,
      "logps/rejected": -16.875,
      "loss": 1.4568,
      "nll_loss": 0.77734375,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 2.890625,
      "rewards/margins": 0.625,
      "rewards/rejected": 2.265625,
      "step": 56
    },
    {
      "epoch": 0.228,
      "grad_norm": 9.34103541373727,
      "learning_rate": 9.022070986717378e-07,
      "logits/chosen": -0.8984375,
      "logits/rejected": -0.90234375,
      "logps/chosen": -7.6875,
      "logps/rejected": -20.75,
      "loss": 1.4427,
      "nll_loss": 0.71875,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.40625,
      "rewards/margins": 1.125,
      "rewards/rejected": 1.28125,
      "step": 57
    },
    {
      "epoch": 0.232,
      "grad_norm": 9.905364057698737,
      "learning_rate": 8.983172782286179e-07,
      "logits/chosen": -0.9140625,
      "logits/rejected": -0.90234375,
      "logps/chosen": -6.46875,
      "logps/rejected": -14.125,
      "loss": 1.4267,
      "nll_loss": 0.69140625,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 2.875,
      "rewards/margins": 0.326171875,
      "rewards/rejected": 2.546875,
      "step": 58
    },
    {
      "epoch": 0.236,
      "grad_norm": 9.50298929918346,
      "learning_rate": 8.943603315859101e-07,
      "logits/chosen": -0.91796875,
      "logits/rejected": -0.92578125,
      "logps/chosen": -6.90625,
      "logps/rejected": -16.25,
      "loss": 1.3527,
      "nll_loss": 0.64453125,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 2.921875,
      "rewards/margins": 1.4375,
      "rewards/rejected": 1.484375,
      "step": 59
    },
    {
      "epoch": 0.24,
      "grad_norm": 9.418798977945164,
      "learning_rate": 8.90336925585864e-07,
      "logits/chosen": -0.9140625,
      "logits/rejected": -0.94140625,
      "logps/chosen": -5.25,
      "logps/rejected": -19.0,
      "loss": 1.4071,
      "nll_loss": 0.546875,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 2.578125,
      "rewards/margins": 1.03125,
      "rewards/rejected": 1.5546875,
      "step": 60
    },
    {
      "epoch": 0.244,
      "grad_norm": 12.471943357949753,
      "learning_rate": 8.862477382707568e-07,
      "logits/chosen": -0.9453125,
      "logits/rejected": -0.9453125,
      "logps/chosen": -7.8125,
      "logps/rejected": -19.5,
      "loss": 1.516,
      "nll_loss": 0.6640625,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.640625,
      "rewards/margins": 0.98828125,
      "rewards/rejected": 1.6640625,
      "step": 61
    },
    {
      "epoch": 0.248,
      "grad_norm": 10.996603171043574,
      "learning_rate": 8.820934587686245e-07,
      "logits/chosen": -0.94921875,
      "logits/rejected": -0.91796875,
      "logps/chosen": -7.21875,
      "logps/rejected": -17.0,
      "loss": 1.4125,
      "nll_loss": 0.703125,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 2.40625,
      "rewards/margins": 0.515625,
      "rewards/rejected": 1.890625,
      "step": 62
    },
    {
      "epoch": 0.252,
      "grad_norm": 10.253437222104349,
      "learning_rate": 8.778747871771291e-07,
      "logits/chosen": -0.91015625,
      "logits/rejected": -0.90234375,
      "logps/chosen": -5.75,
      "logps/rejected": -18.75,
      "loss": 1.3891,
      "nll_loss": 0.55078125,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 2.9375,
      "rewards/margins": 1.140625,
      "rewards/rejected": 1.796875,
      "step": 63
    },
    {
      "epoch": 0.256,
      "grad_norm": 9.86186104574288,
      "learning_rate": 8.735924344455732e-07,
      "logits/chosen": -0.90234375,
      "logits/rejected": -0.91015625,
      "logps/chosen": -6.875,
      "logps/rejected": -18.5,
      "loss": 1.3061,
      "nll_loss": 0.67578125,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 3.015625,
      "rewards/margins": 1.171875,
      "rewards/rejected": 1.84375,
      "step": 64
    },
    {
      "epoch": 0.26,
      "grad_norm": 11.007376728694611,
      "learning_rate": 8.692471222550885e-07,
      "logits/chosen": -0.9453125,
      "logits/rejected": -0.9375,
      "logps/chosen": -7.625,
      "logps/rejected": -23.125,
      "loss": 1.4576,
      "nll_loss": 0.71875,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.625,
      "rewards/margins": 1.4765625,
      "rewards/rejected": 1.1484375,
      "step": 65
    },
    {
      "epoch": 0.264,
      "grad_norm": 10.331552283839107,
      "learning_rate": 8.64839582897015e-07,
      "logits/chosen": -0.9375,
      "logits/rejected": -0.953125,
      "logps/chosen": -7.3125,
      "logps/rejected": -13.5625,
      "loss": 1.3471,
      "nll_loss": 0.6953125,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 2.921875,
      "rewards/margins": 0.384765625,
      "rewards/rejected": 2.546875,
      "step": 66
    },
    {
      "epoch": 0.268,
      "grad_norm": 9.444511843257002,
      "learning_rate": 8.603705591494916e-07,
      "logits/chosen": -0.91015625,
      "logits/rejected": -0.921875,
      "logps/chosen": -6.53125,
      "logps/rejected": -16.5,
      "loss": 1.2306,
      "nll_loss": 0.6796875,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 2.25,
      "rewards/margins": 0.80078125,
      "rewards/rejected": 1.4453125,
      "step": 67
    },
    {
      "epoch": 0.272,
      "grad_norm": 12.577093117314861,
      "learning_rate": 8.5584080415228e-07,
      "logits/chosen": -0.9140625,
      "logits/rejected": -0.91015625,
      "logps/chosen": -5.46875,
      "logps/rejected": -15.5625,
      "loss": 1.2947,
      "nll_loss": 0.5078125,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 2.65625,
      "rewards/margins": 0.87890625,
      "rewards/rejected": 1.78125,
      "step": 68
    },
    {
      "epoch": 0.276,
      "grad_norm": 10.641105295546792,
      "learning_rate": 8.512510812798425e-07,
      "logits/chosen": -0.94140625,
      "logits/rejected": -0.9453125,
      "logps/chosen": -6.3125,
      "logps/rejected": -16.875,
      "loss": 1.2781,
      "nll_loss": 0.578125,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 2.921875,
      "rewards/margins": 1.0546875,
      "rewards/rejected": 1.875,
      "step": 69
    },
    {
      "epoch": 0.28,
      "grad_norm": 12.570856113406187,
      "learning_rate": 8.466021640126945e-07,
      "logits/chosen": -0.953125,
      "logits/rejected": -0.984375,
      "logps/chosen": -5.34375,
      "logps/rejected": -18.625,
      "loss": 1.3463,
      "nll_loss": 0.490234375,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.9375,
      "rewards/margins": 1.25,
      "rewards/rejected": 1.6953125,
      "step": 70
    },
    {
      "epoch": 0.284,
      "grad_norm": 12.634804813000724,
      "learning_rate": 8.418948358070535e-07,
      "logits/chosen": -0.9453125,
      "logits/rejected": -0.9453125,
      "logps/chosen": -7.125,
      "logps/rejected": -17.25,
      "loss": 1.2968,
      "nll_loss": 0.66015625,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 2.703125,
      "rewards/margins": 0.9609375,
      "rewards/rejected": 1.7421875,
      "step": 71
    },
    {
      "epoch": 0.288,
      "grad_norm": 11.739576042187055,
      "learning_rate": 8.37129889962809e-07,
      "logits/chosen": -1.0,
      "logits/rejected": -0.9765625,
      "logps/chosen": -7.4375,
      "logps/rejected": -20.875,
      "loss": 1.3581,
      "nll_loss": 0.67578125,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 2.703125,
      "rewards/margins": 1.3046875,
      "rewards/rejected": 1.40625,
      "step": 72
    },
    {
      "epoch": 0.292,
      "grad_norm": 12.097928995634907,
      "learning_rate": 8.323081294898307e-07,
      "logits/chosen": -0.953125,
      "logits/rejected": -0.9453125,
      "logps/chosen": -5.3125,
      "logps/rejected": -21.5,
      "loss": 1.199,
      "nll_loss": 0.56640625,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.75,
      "rewards/margins": 1.2890625,
      "rewards/rejected": 1.4609375,
      "step": 73
    },
    {
      "epoch": 0.296,
      "grad_norm": 10.920824389039185,
      "learning_rate": 8.274303669726426e-07,
      "logits/chosen": -0.95703125,
      "logits/rejected": -0.94921875,
      "logps/chosen": -5.75,
      "logps/rejected": -18.625,
      "loss": 1.3358,
      "nll_loss": 0.51953125,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 2.609375,
      "rewards/margins": 1.15625,
      "rewards/rejected": 1.4609375,
      "step": 74
    },
    {
      "epoch": 0.3,
      "grad_norm": 14.177002319797584,
      "learning_rate": 8.22497424433482e-07,
      "logits/chosen": -0.93359375,
      "logits/rejected": -0.9296875,
      "logps/chosen": -6.125,
      "logps/rejected": -17.25,
      "loss": 1.2862,
      "nll_loss": 0.61328125,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 2.890625,
      "rewards/margins": 1.3984375,
      "rewards/rejected": 1.5,
      "step": 75
    },
    {
      "epoch": 0.304,
      "grad_norm": 13.6699848015144,
      "learning_rate": 8.175101331937692e-07,
      "logits/chosen": -0.94921875,
      "logits/rejected": -0.95703125,
      "logps/chosen": -6.34375,
      "logps/rejected": -15.625,
      "loss": 1.2559,
      "nll_loss": 0.66015625,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 2.8125,
      "rewards/margins": 1.140625,
      "rewards/rejected": 1.671875,
      "step": 76
    },
    {
      "epoch": 0.308,
      "grad_norm": 13.010408259816026,
      "learning_rate": 8.124693337340092e-07,
      "logits/chosen": -0.99609375,
      "logits/rejected": -0.96484375,
      "logps/chosen": -6.96875,
      "logps/rejected": -21.625,
      "loss": 1.3234,
      "nll_loss": 0.6875,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.890625,
      "rewards/margins": 1.3125,
      "rewards/rejected": 1.5703125,
      "step": 77
    },
    {
      "epoch": 0.312,
      "grad_norm": 14.267160801384883,
      "learning_rate": 8.073758755521504e-07,
      "logits/chosen": -0.99609375,
      "logits/rejected": -0.98828125,
      "logps/chosen": -6.1875,
      "logps/rejected": -23.875,
      "loss": 1.2073,
      "nll_loss": 0.6171875,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 2.921875,
      "rewards/margins": 1.8203125,
      "rewards/rejected": 1.09375,
      "step": 78
    },
    {
      "epoch": 0.316,
      "grad_norm": 10.978973661742327,
      "learning_rate": 8.022306170204231e-07,
      "logits/chosen": -1.0234375,
      "logits/rejected": -1.0,
      "logps/chosen": -5.8125,
      "logps/rejected": -23.5,
      "loss": 1.3188,
      "nll_loss": 0.5625,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 3.09375,
      "rewards/margins": 2.046875,
      "rewards/rejected": 1.0390625,
      "step": 79
    },
    {
      "epoch": 0.32,
      "grad_norm": 11.702933301921057,
      "learning_rate": 7.970344252406831e-07,
      "logits/chosen": -0.9765625,
      "logits/rejected": -0.94921875,
      "logps/chosen": -5.03125,
      "logps/rejected": -18.5,
      "loss": 1.3078,
      "nll_loss": 0.48828125,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 2.90625,
      "rewards/margins": 1.2109375,
      "rewards/rejected": 1.6875,
      "step": 80
    },
    {
      "epoch": 0.324,
      "grad_norm": 12.12062750517865,
      "learning_rate": 7.917881758982836e-07,
      "logits/chosen": -0.953125,
      "logits/rejected": -0.9375,
      "logps/chosen": -6.0,
      "logps/rejected": -15.875,
      "loss": 1.2015,
      "nll_loss": 0.59765625,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 2.65625,
      "rewards/margins": 0.796875,
      "rewards/rejected": 1.859375,
      "step": 81
    },
    {
      "epoch": 0.328,
      "grad_norm": 14.451380371080663,
      "learning_rate": 7.86492753114501e-07,
      "logits/chosen": -1.015625,
      "logits/rejected": -1.015625,
      "logps/chosen": -6.03125,
      "logps/rejected": -24.875,
      "loss": 1.173,
      "nll_loss": 0.55859375,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 2.859375,
      "rewards/margins": 2.078125,
      "rewards/rejected": 0.7734375,
      "step": 82
    },
    {
      "epoch": 0.332,
      "grad_norm": 12.677944938939618,
      "learning_rate": 7.811490492975389e-07,
      "logits/chosen": -0.9609375,
      "logits/rejected": -0.953125,
      "logps/chosen": -4.5,
      "logps/rejected": -20.5,
      "loss": 1.2099,
      "nll_loss": 0.478515625,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.703125,
      "rewards/margins": 1.1484375,
      "rewards/rejected": 1.5625,
      "step": 83
    },
    {
      "epoch": 0.336,
      "grad_norm": 12.912178140489386,
      "learning_rate": 7.757579649921353e-07,
      "logits/chosen": -1.0234375,
      "logits/rejected": -0.99609375,
      "logps/chosen": -4.96875,
      "logps/rejected": -26.0,
      "loss": 1.2007,
      "nll_loss": 0.48828125,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 2.796875,
      "rewards/margins": 1.4765625,
      "rewards/rejected": 1.328125,
      "step": 84
    },
    {
      "epoch": 0.34,
      "grad_norm": 11.95102680724988,
      "learning_rate": 7.703204087277988e-07,
      "logits/chosen": -1.0,
      "logits/rejected": -0.9921875,
      "logps/chosen": -5.125,
      "logps/rejected": -20.75,
      "loss": 1.1374,
      "nll_loss": 0.470703125,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 2.640625,
      "rewards/margins": 1.1953125,
      "rewards/rejected": 1.4453125,
      "step": 85
    },
    {
      "epoch": 0.344,
      "grad_norm": 15.312482138498904,
      "learning_rate": 7.648372968656993e-07,
      "logits/chosen": -0.9921875,
      "logits/rejected": -0.99609375,
      "logps/chosen": -6.15625,
      "logps/rejected": -30.5,
      "loss": 1.1599,
      "nll_loss": 0.55859375,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.515625,
      "rewards/margins": 1.765625,
      "rewards/rejected": 0.75,
      "step": 86
    },
    {
      "epoch": 0.348,
      "grad_norm": 13.90159120685156,
      "learning_rate": 7.593095534442386e-07,
      "logits/chosen": -0.9921875,
      "logits/rejected": -0.984375,
      "logps/chosen": -4.84375,
      "logps/rejected": -19.375,
      "loss": 1.116,
      "nll_loss": 0.52734375,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.9375,
      "rewards/margins": 1.46875,
      "rewards/rejected": 1.46875,
      "step": 87
    },
    {
      "epoch": 0.352,
      "grad_norm": 14.950945255344106,
      "learning_rate": 7.537381100233278e-07,
      "logits/chosen": -1.0078125,
      "logits/rejected": -1.0078125,
      "logps/chosen": -5.0625,
      "logps/rejected": -23.375,
      "loss": 1.2363,
      "nll_loss": 0.482421875,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.640625,
      "rewards/margins": 1.6796875,
      "rewards/rejected": 0.9609375,
      "step": 88
    },
    {
      "epoch": 0.356,
      "grad_norm": 12.117659621877523,
      "learning_rate": 7.481239055273958e-07,
      "logits/chosen": -0.94921875,
      "logits/rejected": -0.9609375,
      "logps/chosen": -4.625,
      "logps/rejected": -12.8125,
      "loss": 1.169,
      "nll_loss": 0.462890625,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 2.828125,
      "rewards/margins": 0.546875,
      "rewards/rejected": 2.28125,
      "step": 89
    },
    {
      "epoch": 0.36,
      "grad_norm": 15.946889828294005,
      "learning_rate": 7.424678860871584e-07,
      "logits/chosen": -1.0078125,
      "logits/rejected": -0.984375,
      "logps/chosen": -6.125,
      "logps/rejected": -28.0,
      "loss": 1.185,
      "nll_loss": 0.58203125,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 2.5625,
      "rewards/margins": 1.703125,
      "rewards/rejected": 0.8515625,
      "step": 90
    },
    {
      "epoch": 0.364,
      "grad_norm": 21.734506154061158,
      "learning_rate": 7.367710048801715e-07,
      "logits/chosen": -1.015625,
      "logits/rejected": -1.015625,
      "logps/chosen": -3.390625,
      "logps/rejected": -19.375,
      "loss": 1.1948,
      "nll_loss": 0.337890625,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.703125,
      "rewards/margins": 1.2421875,
      "rewards/rejected": 1.453125,
      "step": 91
    },
    {
      "epoch": 0.368,
      "grad_norm": 12.733439669594082,
      "learning_rate": 7.31034221970198e-07,
      "logits/chosen": -1.03125,
      "logits/rejected": -1.015625,
      "logps/chosen": -5.3125,
      "logps/rejected": -29.625,
      "loss": 1.1723,
      "nll_loss": 0.55078125,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 2.828125,
      "rewards/margins": 2.015625,
      "rewards/rejected": 0.81640625,
      "step": 92
    },
    {
      "epoch": 0.372,
      "grad_norm": 13.39954488299055,
      "learning_rate": 7.252585041454133e-07,
      "logits/chosen": -1.015625,
      "logits/rejected": -1.0,
      "logps/chosen": -4.4375,
      "logps/rejected": -29.5,
      "loss": 1.1454,
      "nll_loss": 0.443359375,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 2.6875,
      "rewards/margins": 2.15625,
      "rewards/rejected": 0.5390625,
      "step": 93
    },
    {
      "epoch": 0.376,
      "grad_norm": 15.64435560107329,
      "learning_rate": 7.194448247554778e-07,
      "logits/chosen": -1.0,
      "logits/rejected": -0.9921875,
      "logps/chosen": -5.15625,
      "logps/rejected": -20.0,
      "loss": 1.1604,
      "nll_loss": 0.546875,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.8125,
      "rewards/margins": 1.1015625,
      "rewards/rejected": 1.703125,
      "step": 94
    },
    {
      "epoch": 0.38,
      "grad_norm": 17.13096115933346,
      "learning_rate": 7.135941635475036e-07,
      "logits/chosen": -1.015625,
      "logits/rejected": -1.0078125,
      "logps/chosen": -6.8125,
      "logps/rejected": -28.5,
      "loss": 1.2684,
      "nll_loss": 0.6015625,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.953125,
      "rewards/margins": 1.9921875,
      "rewards/rejected": 0.96875,
      "step": 95
    },
    {
      "epoch": 0.384,
      "grad_norm": 16.845492514129948,
      "learning_rate": 7.077075065009433e-07,
      "logits/chosen": -1.0234375,
      "logits/rejected": -0.99609375,
      "logps/chosen": -6.0625,
      "logps/rejected": -22.625,
      "loss": 1.2163,
      "nll_loss": 0.5859375,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.3125,
      "rewards/margins": 1.2265625,
      "rewards/rejected": 1.0859375,
      "step": 96
    },
    {
      "epoch": 0.388,
      "grad_norm": 24.19829824906313,
      "learning_rate": 7.017858456614283e-07,
      "logits/chosen": -1.03125,
      "logits/rejected": -1.0078125,
      "logps/chosen": -4.8125,
      "logps/rejected": -24.125,
      "loss": 1.2011,
      "nll_loss": 0.46875,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 3.0,
      "rewards/margins": 1.75,
      "rewards/rejected": 1.2578125,
      "step": 97
    },
    {
      "epoch": 0.392,
      "grad_norm": 16.528677943117593,
      "learning_rate": 6.958301789735852e-07,
      "logits/chosen": -1.03125,
      "logits/rejected": -1.0234375,
      "logps/chosen": -5.46875,
      "logps/rejected": -26.125,
      "loss": 1.2671,
      "nll_loss": 0.5546875,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.5,
      "rewards/margins": 1.8359375,
      "rewards/rejected": 0.66015625,
      "step": 98
    },
    {
      "epoch": 0.396,
      "grad_norm": 15.542683127815902,
      "learning_rate": 6.898415101128569e-07,
      "logits/chosen": -1.0234375,
      "logits/rejected": -1.03125,
      "logps/chosen": -5.5625,
      "logps/rejected": -18.25,
      "loss": 1.2051,
      "nll_loss": 0.53125,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.796875,
      "rewards/margins": 1.109375,
      "rewards/rejected": 1.6796875,
      "step": 99
    },
    {
      "epoch": 0.4,
      "grad_norm": 16.703597419369395,
      "learning_rate": 6.8382084831636e-07,
      "logits/chosen": -1.0546875,
      "logits/rejected": -1.0234375,
      "logps/chosen": -4.34375,
      "logps/rejected": -30.875,
      "loss": 1.1695,
      "nll_loss": 0.392578125,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 2.46875,
      "rewards/margins": 2.140625,
      "rewards/rejected": 0.333984375,
      "step": 100
    },
    {
      "epoch": 0.404,
      "grad_norm": 20.289662822876423,
      "learning_rate": 6.777692082128024e-07,
      "logits/chosen": -1.046875,
      "logits/rejected": -1.015625,
      "logps/chosen": -6.9375,
      "logps/rejected": -23.125,
      "loss": 1.1355,
      "nll_loss": 0.6328125,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 2.84375,
      "rewards/margins": 1.734375,
      "rewards/rejected": 1.109375,
      "step": 101
    },
    {
      "epoch": 0.408,
      "grad_norm": 10.57908068816297,
      "learning_rate": 6.716876096514943e-07,
      "logits/chosen": -1.0078125,
      "logits/rejected": -1.0234375,
      "logps/chosen": -3.34375,
      "logps/rejected": -35.25,
      "loss": 1.1203,
      "nll_loss": 0.30078125,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.75,
      "rewards/margins": 2.953125,
      "rewards/rejected": -0.2001953125,
      "step": 102
    },
    {
      "epoch": 0.412,
      "grad_norm": 12.448311996517877,
      "learning_rate": 6.655770775304782e-07,
      "logits/chosen": -0.98828125,
      "logits/rejected": -0.98828125,
      "logps/chosen": -5.5,
      "logps/rejected": -13.8125,
      "loss": 1.2512,
      "nll_loss": 0.5625,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 2.90625,
      "rewards/margins": 0.66015625,
      "rewards/rejected": 2.25,
      "step": 103
    },
    {
      "epoch": 0.416,
      "grad_norm": 19.442937641890275,
      "learning_rate": 6.594386416238094e-07,
      "logits/chosen": -0.98046875,
      "logits/rejected": -0.98828125,
      "logps/chosen": -8.0625,
      "logps/rejected": -20.0,
      "loss": 1.1029,
      "nll_loss": 0.75,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 2.546875,
      "rewards/margins": 1.3984375,
      "rewards/rejected": 1.1484375,
      "step": 104
    },
    {
      "epoch": 0.42,
      "grad_norm": 17.597984347295238,
      "learning_rate": 6.532733364080125e-07,
      "logits/chosen": -0.98828125,
      "logits/rejected": -0.97265625,
      "logps/chosen": -5.53125,
      "logps/rejected": -26.125,
      "loss": 1.0465,
      "nll_loss": 0.5078125,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.78125,
      "rewards/margins": 1.84375,
      "rewards/rejected": 0.9375,
      "step": 105
    },
    {
      "epoch": 0.424,
      "grad_norm": 42.65485477575747,
      "learning_rate": 6.470822008877482e-07,
      "logits/chosen": -1.0,
      "logits/rejected": -0.98828125,
      "logps/chosen": -5.9375,
      "logps/rejected": -19.125,
      "loss": 1.1754,
      "nll_loss": 0.53515625,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.875,
      "rewards/margins": 1.390625,
      "rewards/rejected": 1.4921875,
      "step": 106
    },
    {
      "epoch": 0.428,
      "grad_norm": 14.418794180437754,
      "learning_rate": 6.408662784207149e-07,
      "logits/chosen": -1.0078125,
      "logits/rejected": -1.0,
      "logps/chosen": -4.875,
      "logps/rejected": -28.375,
      "loss": 1.1556,
      "nll_loss": 0.51171875,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.71875,
      "rewards/margins": 1.875,
      "rewards/rejected": 0.84375,
      "step": 107
    },
    {
      "epoch": 0.432,
      "grad_norm": 24.069184880501325,
      "learning_rate": 6.346266165418172e-07,
      "logits/chosen": -0.9765625,
      "logits/rejected": -0.95703125,
      "logps/chosen": -4.71875,
      "logps/rejected": -23.0,
      "loss": 1.1594,
      "nll_loss": 0.48046875,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 3.15625,
      "rewards/margins": 1.875,
      "rewards/rejected": 1.2734375,
      "step": 108
    },
    {
      "epoch": 0.436,
      "grad_norm": 33.99711892618498,
      "learning_rate": 6.283642667866315e-07,
      "logits/chosen": -1.03125,
      "logits/rejected": -0.98046875,
      "logps/chosen": -6.28125,
      "logps/rejected": -27.375,
      "loss": 1.1293,
      "nll_loss": 0.58984375,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 2.5625,
      "rewards/margins": 1.625,
      "rewards/rejected": 0.9375,
      "step": 109
    },
    {
      "epoch": 0.44,
      "grad_norm": 35.15701910686572,
      "learning_rate": 6.220802845141957e-07,
      "logits/chosen": -1.03125,
      "logits/rejected": -1.03125,
      "logps/chosen": -5.71875,
      "logps/rejected": -37.25,
      "loss": 1.1224,
      "nll_loss": 0.51953125,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.578125,
      "rewards/margins": 2.609375,
      "rewards/rejected": -0.03564453125,
      "step": 110
    },
    {
      "epoch": 0.444,
      "grad_norm": 16.617005010321233,
      "learning_rate": 6.157757287291556e-07,
      "logits/chosen": -1.015625,
      "logits/rejected": -1.015625,
      "logps/chosen": -3.859375,
      "logps/rejected": -31.75,
      "loss": 1.0481,
      "nll_loss": 0.40234375,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 2.6875,
      "rewards/margins": 2.6875,
      "rewards/rejected": 0.0074462890625,
      "step": 111
    },
    {
      "epoch": 0.448,
      "grad_norm": 12.591607226736722,
      "learning_rate": 6.094516619032975e-07,
      "logits/chosen": -1.046875,
      "logits/rejected": -1.0390625,
      "logps/chosen": -5.125,
      "logps/rejected": -26.5,
      "loss": 1.0584,
      "nll_loss": 0.51171875,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 2.71875,
      "rewards/margins": 1.90625,
      "rewards/rejected": 0.82421875,
      "step": 112
    },
    {
      "epoch": 0.452,
      "grad_norm": 12.509284400507388,
      "learning_rate": 6.03109149796494e-07,
      "logits/chosen": -1.0546875,
      "logits/rejected": -1.046875,
      "logps/chosen": -3.984375,
      "logps/rejected": -23.0,
      "loss": 1.0653,
      "nll_loss": 0.390625,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 2.859375,
      "rewards/margins": 1.7890625,
      "rewards/rejected": 1.0625,
      "step": 113
    },
    {
      "epoch": 0.456,
      "grad_norm": 33.81401585194403,
      "learning_rate": 5.967492612770997e-07,
      "logits/chosen": -1.03125,
      "logits/rejected": -1.015625,
      "logps/chosen": -6.65625,
      "logps/rejected": -28.125,
      "loss": 1.0973,
      "nll_loss": 0.6640625,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 2.625,
      "rewards/margins": 2.015625,
      "rewards/rejected": 0.6015625,
      "step": 114
    },
    {
      "epoch": 0.46,
      "grad_norm": 15.815532311957057,
      "learning_rate": 5.90373068141819e-07,
      "logits/chosen": -1.0,
      "logits/rejected": -0.9765625,
      "logps/chosen": -6.875,
      "logps/rejected": -29.125,
      "loss": 1.2058,
      "nll_loss": 0.69140625,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.390625,
      "rewards/margins": 2.015625,
      "rewards/rejected": 0.375,
      "step": 115
    },
    {
      "epoch": 0.464,
      "grad_norm": 22.102704341248568,
      "learning_rate": 5.839816449350824e-07,
      "logits/chosen": -0.9921875,
      "logits/rejected": -0.9921875,
      "logps/chosen": -5.78125,
      "logps/rejected": -23.625,
      "loss": 1.1099,
      "nll_loss": 0.5703125,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 3.015625,
      "rewards/margins": 1.5390625,
      "rewards/rejected": 1.46875,
      "step": 116
    },
    {
      "epoch": 0.468,
      "grad_norm": 16.24899670304244,
      "learning_rate": 5.775760687679603e-07,
      "logits/chosen": -1.0078125,
      "logits/rejected": -0.984375,
      "logps/chosen": -4.28125,
      "logps/rejected": -23.125,
      "loss": 1.0259,
      "nll_loss": 0.41015625,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.78125,
      "rewards/margins": 1.765625,
      "rewards/rejected": 1.0234375,
      "step": 117
    },
    {
      "epoch": 0.472,
      "grad_norm": 14.705074940426503,
      "learning_rate": 5.711574191366427e-07,
      "logits/chosen": -1.0234375,
      "logits/rejected": -1.0234375,
      "logps/chosen": -3.203125,
      "logps/rejected": -25.875,
      "loss": 1.1279,
      "nll_loss": 0.330078125,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 3.0,
      "rewards/margins": 1.796875,
      "rewards/rejected": 1.1953125,
      "step": 118
    },
    {
      "epoch": 0.476,
      "grad_norm": 20.221686205664327,
      "learning_rate": 5.647267777405177e-07,
      "logits/chosen": -1.0234375,
      "logits/rejected": -1.03125,
      "logps/chosen": -4.40625,
      "logps/rejected": -23.375,
      "loss": 1.1415,
      "nll_loss": 0.478515625,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.578125,
      "rewards/margins": 1.3984375,
      "rewards/rejected": 1.1796875,
      "step": 119
    },
    {
      "epoch": 0.48,
      "grad_norm": 24.368984032622865,
      "learning_rate": 5.582852282998795e-07,
      "logits/chosen": -1.0234375,
      "logits/rejected": -1.03125,
      "logps/chosen": -5.40625,
      "logps/rejected": -23.125,
      "loss": 1.0667,
      "nll_loss": 0.474609375,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.84375,
      "rewards/margins": 1.546875,
      "rewards/rejected": 1.296875,
      "step": 120
    },
    {
      "epoch": 0.484,
      "grad_norm": 22.432812053652427,
      "learning_rate": 5.518338563732944e-07,
      "logits/chosen": -1.046875,
      "logits/rejected": -1.0390625,
      "logps/chosen": -4.90625,
      "logps/rejected": -23.625,
      "loss": 1.0827,
      "nll_loss": 0.54296875,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.546875,
      "rewards/margins": 1.6484375,
      "rewards/rejected": 0.90625,
      "step": 121
    },
    {
      "epoch": 0.488,
      "grad_norm": 16.17860106216787,
      "learning_rate": 5.453737491746572e-07,
      "logits/chosen": -1.0234375,
      "logits/rejected": -1.03125,
      "logps/chosen": -3.625,
      "logps/rejected": -22.875,
      "loss": 1.0798,
      "nll_loss": 0.375,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 3.046875,
      "rewards/margins": 1.609375,
      "rewards/rejected": 1.4296875,
      "step": 122
    },
    {
      "epoch": 0.492,
      "grad_norm": 24.70109578919281,
      "learning_rate": 5.389059953899698e-07,
      "logits/chosen": -1.09375,
      "logits/rejected": -1.0703125,
      "logps/chosen": -4.53125,
      "logps/rejected": -31.375,
      "loss": 1.2218,
      "nll_loss": 0.451171875,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 3.109375,
      "rewards/margins": 2.34375,
      "rewards/rejected": 0.765625,
      "step": 123
    },
    {
      "epoch": 0.496,
      "grad_norm": 31.00791322114362,
      "learning_rate": 5.324316849938714e-07,
      "logits/chosen": -0.98046875,
      "logits/rejected": -0.97265625,
      "logps/chosen": -6.375,
      "logps/rejected": -21.75,
      "loss": 1.166,
      "nll_loss": 0.57421875,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.71875,
      "rewards/margins": 1.5703125,
      "rewards/rejected": 1.15625,
      "step": 124
    },
    {
      "epoch": 0.5,
      "grad_norm": 37.45373935605177,
      "learning_rate": 5.259519090659498e-07,
      "logits/chosen": -1.0546875,
      "logits/rejected": -1.046875,
      "logps/chosen": -4.40625,
      "logps/rejected": -25.5,
      "loss": 1.0699,
      "nll_loss": 0.392578125,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.890625,
      "rewards/margins": 1.703125,
      "rewards/rejected": 1.1953125,
      "step": 125
    },
    {
      "epoch": 0.504,
      "grad_norm": 12.712491511773225,
      "learning_rate": 5.194677596068688e-07,
      "logits/chosen": -1.0078125,
      "logits/rejected": -1.03125,
      "logps/chosen": -5.84375,
      "logps/rejected": -21.375,
      "loss": 1.1111,
      "nll_loss": 0.55859375,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 2.65625,
      "rewards/margins": 1.1328125,
      "rewards/rejected": 1.5234375,
      "step": 126
    },
    {
      "epoch": 0.508,
      "grad_norm": 14.71397944686033,
      "learning_rate": 5.129803293543391e-07,
      "logits/chosen": -1.0546875,
      "logits/rejected": -1.046875,
      "logps/chosen": -4.375,
      "logps/rejected": -28.875,
      "loss": 1.0949,
      "nll_loss": 0.41015625,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.984375,
      "rewards/margins": 2.390625,
      "rewards/rejected": 0.58984375,
      "step": 127
    },
    {
      "epoch": 0.512,
      "grad_norm": 20.40214155927554,
      "learning_rate": 5.064907115989654e-07,
      "logits/chosen": -1.078125,
      "logits/rejected": -1.09375,
      "logps/chosen": -5.375,
      "logps/rejected": -30.25,
      "loss": 1.0997,
      "nll_loss": 0.46484375,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.984375,
      "rewards/margins": 2.84375,
      "rewards/rejected": 0.1357421875,
      "step": 128
    },
    {
      "epoch": 0.516,
      "grad_norm": 20.076566430152866,
      "learning_rate": 5e-07,
      "logits/chosen": -1.0546875,
      "logits/rejected": -1.0546875,
      "logps/chosen": -5.84375,
      "logps/rejected": -31.5,
      "loss": 1.1843,
      "nll_loss": 0.5703125,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.484375,
      "rewards/margins": 1.984375,
      "rewards/rejected": 0.5078125,
      "step": 129
    },
    {
      "epoch": 0.52,
      "grad_norm": 15.69512072569519,
      "learning_rate": 4.935092884010346e-07,
      "logits/chosen": -1.0234375,
      "logits/rejected": -1.015625,
      "logps/chosen": -4.6875,
      "logps/rejected": -24.75,
      "loss": 1.0591,
      "nll_loss": 0.4609375,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.84375,
      "rewards/margins": 1.8828125,
      "rewards/rejected": 0.96484375,
      "step": 130
    },
    {
      "epoch": 0.524,
      "grad_norm": 15.184658306188565,
      "learning_rate": 4.870196706456609e-07,
      "logits/chosen": -1.0625,
      "logits/rejected": -1.09375,
      "logps/chosen": -5.90625,
      "logps/rejected": -29.5,
      "loss": 1.1098,
      "nll_loss": 0.58203125,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 2.828125,
      "rewards/margins": 1.7109375,
      "rewards/rejected": 1.1171875,
      "step": 131
    },
    {
      "epoch": 0.528,
      "grad_norm": 30.109778759207604,
      "learning_rate": 4.805322403931311e-07,
      "logits/chosen": -1.109375,
      "logits/rejected": -1.1015625,
      "logps/chosen": -4.78125,
      "logps/rejected": -28.25,
      "loss": 1.0583,
      "nll_loss": 0.41796875,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 3.078125,
      "rewards/margins": 2.765625,
      "rewards/rejected": 0.30859375,
      "step": 132
    },
    {
      "epoch": 0.532,
      "grad_norm": 19.180407490964914,
      "learning_rate": 4.7404809093405017e-07,
      "logits/chosen": -1.0703125,
      "logits/rejected": -1.0703125,
      "logps/chosen": -4.5625,
      "logps/rejected": -24.5,
      "loss": 1.0781,
      "nll_loss": 0.3984375,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.6875,
      "rewards/margins": 1.75,
      "rewards/rejected": 0.9375,
      "step": 133
    },
    {
      "epoch": 0.536,
      "grad_norm": 11.865585536243879,
      "learning_rate": 4.675683150061284e-07,
      "logits/chosen": -1.078125,
      "logits/rejected": -1.0390625,
      "logps/chosen": -4.65625,
      "logps/rejected": -27.75,
      "loss": 1.1264,
      "nll_loss": 0.470703125,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 3.140625,
      "rewards/margins": 1.9375,
      "rewards/rejected": 1.203125,
      "step": 134
    },
    {
      "epoch": 0.54,
      "grad_norm": 14.216123496302846,
      "learning_rate": 4.6109400461003003e-07,
      "logits/chosen": -1.0234375,
      "logits/rejected": -1.0234375,
      "logps/chosen": -5.34375,
      "logps/rejected": -23.5,
      "loss": 1.0383,
      "nll_loss": 0.453125,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 3.3125,
      "rewards/margins": 2.25,
      "rewards/rejected": 1.0703125,
      "step": 135
    },
    {
      "epoch": 0.544,
      "grad_norm": 28.567567684765535,
      "learning_rate": 4.5462625082534286e-07,
      "logits/chosen": -1.078125,
      "logits/rejected": -1.0703125,
      "logps/chosen": -3.9375,
      "logps/rejected": -28.125,
      "loss": 0.9913,
      "nll_loss": 0.388671875,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 2.875,
      "rewards/margins": 2.53125,
      "rewards/rejected": 0.353515625,
      "step": 136
    },
    {
      "epoch": 0.548,
      "grad_norm": 19.382351209373788,
      "learning_rate": 4.481661436267057e-07,
      "logits/chosen": -1.0703125,
      "logits/rejected": -1.078125,
      "logps/chosen": -4.65625,
      "logps/rejected": -33.5,
      "loss": 1.1359,
      "nll_loss": 0.42578125,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.828125,
      "rewards/margins": 2.328125,
      "rewards/rejected": 0.5078125,
      "step": 137
    },
    {
      "epoch": 0.552,
      "grad_norm": 28.26009731578296,
      "learning_rate": 4.4171477170012043e-07,
      "logits/chosen": -1.0546875,
      "logits/rejected": -1.046875,
      "logps/chosen": -4.21875,
      "logps/rejected": -32.0,
      "loss": 1.1338,
      "nll_loss": 0.380859375,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 2.953125,
      "rewards/margins": 2.578125,
      "rewards/rejected": 0.37890625,
      "step": 138
    },
    {
      "epoch": 0.556,
      "grad_norm": 48.13783869940889,
      "learning_rate": 4.352732222594823e-07,
      "logits/chosen": -1.046875,
      "logits/rejected": -1.03125,
      "logps/chosen": -5.1875,
      "logps/rejected": -25.625,
      "loss": 1.1776,
      "nll_loss": 0.466796875,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 2.859375,
      "rewards/margins": 1.8515625,
      "rewards/rejected": 1.0078125,
      "step": 139
    },
    {
      "epoch": 0.56,
      "grad_norm": 23.304268399995618,
      "learning_rate": 4.2884258086335745e-07,
      "logits/chosen": -1.0390625,
      "logits/rejected": -1.0390625,
      "logps/chosen": -5.65625,
      "logps/rejected": -18.5,
      "loss": 1.1203,
      "nll_loss": 0.490234375,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 3.03125,
      "rewards/margins": 1.421875,
      "rewards/rejected": 1.6015625,
      "step": 140
    },
    {
      "epoch": 0.564,
      "grad_norm": 19.29343324391366,
      "learning_rate": 4.224239312320398e-07,
      "logits/chosen": -1.078125,
      "logits/rejected": -1.0703125,
      "logps/chosen": -7.90625,
      "logps/rejected": -29.375,
      "loss": 1.1849,
      "nll_loss": 0.67578125,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.5,
      "rewards/margins": 1.9375,
      "rewards/rejected": 0.5625,
      "step": 141
    },
    {
      "epoch": 0.568,
      "grad_norm": 17.35581169203161,
      "learning_rate": 4.160183550649175e-07,
      "logits/chosen": -1.0546875,
      "logits/rejected": -1.0546875,
      "logps/chosen": -4.65625,
      "logps/rejected": -28.25,
      "loss": 1.0433,
      "nll_loss": 0.48046875,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 2.796875,
      "rewards/margins": 2.03125,
      "rewards/rejected": 0.76953125,
      "step": 142
    },
    {
      "epoch": 0.572,
      "grad_norm": 31.43926475473609,
      "learning_rate": 4.09626931858181e-07,
      "logits/chosen": -1.0625,
      "logits/rejected": -1.0546875,
      "logps/chosen": -4.21875,
      "logps/rejected": -19.125,
      "loss": 1.112,
      "nll_loss": 0.41015625,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.96875,
      "rewards/margins": 1.3515625,
      "rewards/rejected": 1.625,
      "step": 143
    },
    {
      "epoch": 0.576,
      "grad_norm": 19.985361598611938,
      "learning_rate": 4.0325073872290015e-07,
      "logits/chosen": -1.0234375,
      "logits/rejected": -1.015625,
      "logps/chosen": -6.6875,
      "logps/rejected": -26.375,
      "loss": 1.1101,
      "nll_loss": 0.5859375,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.859375,
      "rewards/margins": 2.328125,
      "rewards/rejected": 0.52734375,
      "step": 144
    },
    {
      "epoch": 0.58,
      "grad_norm": 35.012979635813075,
      "learning_rate": 3.96890850203506e-07,
      "logits/chosen": -1.0625,
      "logits/rejected": -1.0625,
      "logps/chosen": -6.375,
      "logps/rejected": -26.625,
      "loss": 1.0092,
      "nll_loss": 0.59765625,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 3.21875,
      "rewards/margins": 2.265625,
      "rewards/rejected": 0.94921875,
      "step": 145
    },
    {
      "epoch": 0.584,
      "grad_norm": 20.26642224735176,
      "learning_rate": 3.9054833809670265e-07,
      "logits/chosen": -1.0546875,
      "logits/rejected": -1.0546875,
      "logps/chosen": -4.625,
      "logps/rejected": -29.5,
      "loss": 1.0404,
      "nll_loss": 0.4140625,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.75,
      "rewards/margins": 2.5,
      "rewards/rejected": 0.2392578125,
      "step": 146
    },
    {
      "epoch": 0.588,
      "grad_norm": 13.154350173457996,
      "learning_rate": 3.8422427127084436e-07,
      "logits/chosen": -1.078125,
      "logits/rejected": -1.0625,
      "logps/chosen": -6.53125,
      "logps/rejected": -31.875,
      "loss": 0.9968,
      "nll_loss": 0.58203125,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.921875,
      "rewards/margins": 2.59375,
      "rewards/rejected": 0.32421875,
      "step": 147
    },
    {
      "epoch": 0.592,
      "grad_norm": 23.129629439239768,
      "learning_rate": 3.779197154858044e-07,
      "logits/chosen": -1.0546875,
      "logits/rejected": -1.0390625,
      "logps/chosen": -5.21875,
      "logps/rejected": -23.375,
      "loss": 0.9492,
      "nll_loss": 0.4921875,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 3.09375,
      "rewards/margins": 1.7265625,
      "rewards/rejected": 1.3671875,
      "step": 148
    },
    {
      "epoch": 0.596,
      "grad_norm": 14.081159200332113,
      "learning_rate": 3.716357332133686e-07,
      "logits/chosen": -1.1171875,
      "logits/rejected": -1.078125,
      "logps/chosen": -5.65625,
      "logps/rejected": -30.5,
      "loss": 1.0858,
      "nll_loss": 0.58984375,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.828125,
      "rewards/margins": 2.0625,
      "rewards/rejected": 0.76953125,
      "step": 149
    },
    {
      "epoch": 0.6,
      "grad_norm": 30.59966116072142,
      "learning_rate": 3.653733834581827e-07,
      "logits/chosen": -1.09375,
      "logits/rejected": -1.078125,
      "logps/chosen": -4.0,
      "logps/rejected": -26.125,
      "loss": 1.0947,
      "nll_loss": 0.373046875,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.765625,
      "rewards/margins": 2.21875,
      "rewards/rejected": 0.546875,
      "step": 150
    },
    {
      "epoch": 0.604,
      "grad_norm": 14.36839336730757,
      "learning_rate": 3.591337215792851e-07,
      "logits/chosen": -1.09375,
      "logits/rejected": -1.109375,
      "logps/chosen": -4.5625,
      "logps/rejected": -27.5,
      "loss": 1.085,
      "nll_loss": 0.45703125,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 2.90625,
      "rewards/margins": 1.8984375,
      "rewards/rejected": 1.0,
      "step": 151
    },
    {
      "epoch": 0.608,
      "grad_norm": 25.95175246611989,
      "learning_rate": 3.5291779911225185e-07,
      "logits/chosen": -1.0859375,
      "logits/rejected": -1.0859375,
      "logps/chosen": -4.5,
      "logps/rejected": -30.75,
      "loss": 1.0704,
      "nll_loss": 0.427734375,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 2.984375,
      "rewards/margins": 2.265625,
      "rewards/rejected": 0.71484375,
      "step": 152
    },
    {
      "epoch": 0.612,
      "grad_norm": 30.341569831825527,
      "learning_rate": 3.4672666359198757e-07,
      "logits/chosen": -1.078125,
      "logits/rejected": -1.0625,
      "logps/chosen": -4.21875,
      "logps/rejected": -27.75,
      "loss": 1.0871,
      "nll_loss": 0.453125,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 3.265625,
      "rewards/margins": 2.359375,
      "rewards/rejected": 0.91015625,
      "step": 153
    },
    {
      "epoch": 0.616,
      "grad_norm": 15.724055467705009,
      "learning_rate": 3.405613583761907e-07,
      "logits/chosen": -1.0546875,
      "logits/rejected": -1.0390625,
      "logps/chosen": -5.53125,
      "logps/rejected": -22.25,
      "loss": 1.0548,
      "nll_loss": 0.5625,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 3.078125,
      "rewards/margins": 1.5625,
      "rewards/rejected": 1.5234375,
      "step": 154
    },
    {
      "epoch": 0.62,
      "grad_norm": 23.186267615438663,
      "learning_rate": 3.3442292246952186e-07,
      "logits/chosen": -1.109375,
      "logits/rejected": -1.0859375,
      "logps/chosen": -5.71875,
      "logps/rejected": -33.75,
      "loss": 1.0691,
      "nll_loss": 0.5390625,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.5,
      "rewards/margins": 2.390625,
      "rewards/rejected": 0.10595703125,
      "step": 155
    },
    {
      "epoch": 0.624,
      "grad_norm": 19.08999867823956,
      "learning_rate": 3.283123903485059e-07,
      "logits/chosen": -1.0546875,
      "logits/rejected": -1.0390625,
      "logps/chosen": -6.15625,
      "logps/rejected": -28.625,
      "loss": 1.1063,
      "nll_loss": 0.6015625,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 3.046875,
      "rewards/margins": 2.03125,
      "rewards/rejected": 1.015625,
      "step": 156
    },
    {
      "epoch": 0.628,
      "grad_norm": 24.530260893855512,
      "learning_rate": 3.222307917871977e-07,
      "logits/chosen": -1.1015625,
      "logits/rejected": -1.078125,
      "logps/chosen": -5.25,
      "logps/rejected": -36.0,
      "loss": 1.1445,
      "nll_loss": 0.46875,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.53125,
      "rewards/margins": 2.90625,
      "rewards/rejected": -0.376953125,
      "step": 157
    },
    {
      "epoch": 0.632,
      "grad_norm": 42.24994864204084,
      "learning_rate": 3.161791516836399e-07,
      "logits/chosen": -1.078125,
      "logits/rejected": -1.0859375,
      "logps/chosen": -6.28125,
      "logps/rejected": -32.25,
      "loss": 1.0581,
      "nll_loss": 0.55859375,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 2.8125,
      "rewards/margins": 2.328125,
      "rewards/rejected": 0.484375,
      "step": 158
    },
    {
      "epoch": 0.636,
      "grad_norm": 14.715119673174955,
      "learning_rate": 3.1015848988714307e-07,
      "logits/chosen": -1.0546875,
      "logits/rejected": -1.0390625,
      "logps/chosen": -6.5,
      "logps/rejected": -33.75,
      "loss": 0.9939,
      "nll_loss": 0.609375,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 3.203125,
      "rewards/margins": 2.90625,
      "rewards/rejected": 0.296875,
      "step": 159
    },
    {
      "epoch": 0.64,
      "grad_norm": 20.14211721361461,
      "learning_rate": 3.0416982102641487e-07,
      "logits/chosen": -1.0625,
      "logits/rejected": -1.0625,
      "logps/chosen": -5.625,
      "logps/rejected": -27.25,
      "loss": 1.0876,
      "nll_loss": 0.5703125,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 3.296875,
      "rewards/margins": 2.390625,
      "rewards/rejected": 0.8984375,
      "step": 160
    },
    {
      "epoch": 0.644,
      "grad_norm": 15.676197977531485,
      "learning_rate": 2.982141543385717e-07,
      "logits/chosen": -1.0703125,
      "logits/rejected": -1.0625,
      "logps/chosen": -4.75,
      "logps/rejected": -25.25,
      "loss": 1.0416,
      "nll_loss": 0.490234375,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 2.953125,
      "rewards/margins": 1.8515625,
      "rewards/rejected": 1.1015625,
      "step": 161
    },
    {
      "epoch": 0.648,
      "grad_norm": 18.922807692403683,
      "learning_rate": 2.922924934990568e-07,
      "logits/chosen": -1.03125,
      "logits/rejected": -1.0390625,
      "logps/chosen": -4.21875,
      "logps/rejected": -28.25,
      "loss": 1.0214,
      "nll_loss": 0.427734375,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 2.578125,
      "rewards/margins": 2.1875,
      "rewards/rejected": 0.3984375,
      "step": 162
    },
    {
      "epoch": 0.652,
      "grad_norm": 15.211250498950244,
      "learning_rate": 2.864058364524964e-07,
      "logits/chosen": -1.0625,
      "logits/rejected": -1.03125,
      "logps/chosen": -3.875,
      "logps/rejected": -26.625,
      "loss": 1.0185,
      "nll_loss": 0.39453125,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 2.984375,
      "rewards/margins": 2.453125,
      "rewards/rejected": 0.53515625,
      "step": 163
    },
    {
      "epoch": 0.656,
      "grad_norm": 32.56527905487464,
      "learning_rate": 2.8055517524452215e-07,
      "logits/chosen": -1.1171875,
      "logits/rejected": -1.09375,
      "logps/chosen": -4.875,
      "logps/rejected": -27.125,
      "loss": 0.9807,
      "nll_loss": 0.4453125,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.6875,
      "rewards/margins": 2.09375,
      "rewards/rejected": 0.58984375,
      "step": 164
    },
    {
      "epoch": 0.66,
      "grad_norm": 10.018587276474245,
      "learning_rate": 2.7474149585458664e-07,
      "logits/chosen": -1.09375,
      "logits/rejected": -1.1171875,
      "logps/chosen": -3.390625,
      "logps/rejected": -32.75,
      "loss": 1.0211,
      "nll_loss": 0.310546875,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 2.484375,
      "rewards/margins": 2.421875,
      "rewards/rejected": 0.0595703125,
      "step": 165
    },
    {
      "epoch": 0.664,
      "grad_norm": 37.10528373181994,
      "learning_rate": 2.6896577802980184e-07,
      "logits/chosen": -1.078125,
      "logits/rejected": -1.078125,
      "logps/chosen": -3.296875,
      "logps/rejected": -33.25,
      "loss": 1.0286,
      "nll_loss": 0.310546875,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 2.734375,
      "rewards/margins": 2.71875,
      "rewards/rejected": 0.0106201171875,
      "step": 166
    },
    {
      "epoch": 0.668,
      "grad_norm": 17.20600574229811,
      "learning_rate": 2.632289951198285e-07,
      "logits/chosen": -1.0859375,
      "logits/rejected": -1.109375,
      "logps/chosen": -5.1875,
      "logps/rejected": -30.375,
      "loss": 1.1248,
      "nll_loss": 0.47265625,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 3.078125,
      "rewards/margins": 2.46875,
      "rewards/rejected": 0.609375,
      "step": 167
    },
    {
      "epoch": 0.672,
      "grad_norm": 38.3608360818198,
      "learning_rate": 2.575321139128417e-07,
      "logits/chosen": -1.1171875,
      "logits/rejected": -1.1015625,
      "logps/chosen": -5.25,
      "logps/rejected": -26.75,
      "loss": 0.9945,
      "nll_loss": 0.49609375,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.984375,
      "rewards/margins": 1.9765625,
      "rewards/rejected": 1.0,
      "step": 168
    },
    {
      "epoch": 0.676,
      "grad_norm": 17.759279183503754,
      "learning_rate": 2.5187609447260417e-07,
      "logits/chosen": -1.1015625,
      "logits/rejected": -1.09375,
      "logps/chosen": -4.375,
      "logps/rejected": -30.25,
      "loss": 1.0025,
      "nll_loss": 0.439453125,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 2.96875,
      "rewards/margins": 2.625,
      "rewards/rejected": 0.3515625,
      "step": 169
    },
    {
      "epoch": 0.68,
      "grad_norm": 14.507361507718214,
      "learning_rate": 2.462618899766722e-07,
      "logits/chosen": -1.0546875,
      "logits/rejected": -1.078125,
      "logps/chosen": -6.21875,
      "logps/rejected": -32.75,
      "loss": 1.0907,
      "nll_loss": 0.60546875,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.359375,
      "rewards/margins": 2.078125,
      "rewards/rejected": 0.2734375,
      "step": 170
    },
    {
      "epoch": 0.684,
      "grad_norm": 22.962611189789666,
      "learning_rate": 2.406904465557614e-07,
      "logits/chosen": -1.109375,
      "logits/rejected": -1.1171875,
      "logps/chosen": -4.5625,
      "logps/rejected": -28.25,
      "loss": 1.0443,
      "nll_loss": 0.4296875,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 2.671875,
      "rewards/margins": 1.859375,
      "rewards/rejected": 0.8203125,
      "step": 171
    },
    {
      "epoch": 0.688,
      "grad_norm": 18.306407029166536,
      "learning_rate": 2.3516270313430081e-07,
      "logits/chosen": -1.1015625,
      "logits/rejected": -1.0703125,
      "logps/chosen": -7.21875,
      "logps/rejected": -33.0,
      "loss": 1.0611,
      "nll_loss": 0.6484375,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.9375,
      "rewards/margins": 2.796875,
      "rewards/rejected": 0.140625,
      "step": 172
    },
    {
      "epoch": 0.692,
      "grad_norm": 12.791171107272646,
      "learning_rate": 2.2967959127220137e-07,
      "logits/chosen": -1.0859375,
      "logits/rejected": -1.0703125,
      "logps/chosen": -3.765625,
      "logps/rejected": -26.25,
      "loss": 0.9565,
      "nll_loss": 0.3515625,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 3.125,
      "rewards/margins": 2.5,
      "rewards/rejected": 0.6328125,
      "step": 173
    },
    {
      "epoch": 0.696,
      "grad_norm": 24.89717076883767,
      "learning_rate": 2.242420350078647e-07,
      "logits/chosen": -1.0703125,
      "logits/rejected": -1.0703125,
      "logps/chosen": -5.1875,
      "logps/rejected": -27.75,
      "loss": 1.0301,
      "nll_loss": 0.486328125,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 3.015625,
      "rewards/margins": 2.421875,
      "rewards/rejected": 0.59765625,
      "step": 174
    },
    {
      "epoch": 0.7,
      "grad_norm": 30.549643731351598,
      "learning_rate": 2.1885095070246113e-07,
      "logits/chosen": -1.1015625,
      "logits/rejected": -1.109375,
      "logps/chosen": -4.53125,
      "logps/rejected": -34.0,
      "loss": 1.1014,
      "nll_loss": 0.439453125,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 3.046875,
      "rewards/margins": 2.8125,
      "rewards/rejected": 0.2412109375,
      "step": 175
    },
    {
      "epoch": 0.704,
      "grad_norm": 42.66332171882467,
      "learning_rate": 2.1350724688549904e-07,
      "logits/chosen": -1.15625,
      "logits/rejected": -1.1484375,
      "logps/chosen": -5.46875,
      "logps/rejected": -31.5,
      "loss": 1.1445,
      "nll_loss": 0.5625,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 2.390625,
      "rewards/margins": 2.09375,
      "rewards/rejected": 0.306640625,
      "step": 176
    },
    {
      "epoch": 0.708,
      "grad_norm": 19.430831101872652,
      "learning_rate": 2.0821182410171633e-07,
      "logits/chosen": -1.109375,
      "logits/rejected": -1.125,
      "logps/chosen": -3.5625,
      "logps/rejected": -32.75,
      "loss": 1.0655,
      "nll_loss": 0.328125,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.390625,
      "rewards/margins": 2.171875,
      "rewards/rejected": 0.2099609375,
      "step": 177
    },
    {
      "epoch": 0.712,
      "grad_norm": 40.306067494211455,
      "learning_rate": 2.0296557475931687e-07,
      "logits/chosen": -1.09375,
      "logits/rejected": -1.09375,
      "logps/chosen": -6.21875,
      "logps/rejected": -28.0,
      "loss": 1.1514,
      "nll_loss": 0.59375,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.84375,
      "rewards/margins": 1.65625,
      "rewards/rejected": 1.1796875,
      "step": 178
    },
    {
      "epoch": 0.716,
      "grad_norm": 21.550210732610292,
      "learning_rate": 1.9776938297957686e-07,
      "logits/chosen": -1.0625,
      "logits/rejected": -1.03125,
      "logps/chosen": -3.140625,
      "logps/rejected": -18.75,
      "loss": 1.0927,
      "nll_loss": 0.33203125,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 3.0,
      "rewards/margins": 1.4609375,
      "rewards/rejected": 1.546875,
      "step": 179
    },
    {
      "epoch": 0.72,
      "grad_norm": 16.05057780811457,
      "learning_rate": 1.9262412444784958e-07,
      "logits/chosen": -1.0078125,
      "logits/rejected": -1.0234375,
      "logps/chosen": -7.0,
      "logps/rejected": -21.25,
      "loss": 1.0314,
      "nll_loss": 0.6875,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.75,
      "rewards/margins": 1.4765625,
      "rewards/rejected": 1.2578125,
      "step": 180
    },
    {
      "epoch": 0.724,
      "grad_norm": 22.311212494168778,
      "learning_rate": 1.8753066626599084e-07,
      "logits/chosen": -1.1015625,
      "logits/rejected": -1.09375,
      "logps/chosen": -3.75,
      "logps/rejected": -34.25,
      "loss": 1.1563,
      "nll_loss": 0.38671875,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.84375,
      "rewards/margins": 2.234375,
      "rewards/rejected": 0.6015625,
      "step": 181
    },
    {
      "epoch": 0.728,
      "grad_norm": 24.732581587145912,
      "learning_rate": 1.8248986680623074e-07,
      "logits/chosen": -1.0625,
      "logits/rejected": -1.0625,
      "logps/chosen": -5.0625,
      "logps/rejected": -23.625,
      "loss": 1.1566,
      "nll_loss": 0.5078125,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 2.59375,
      "rewards/margins": 1.546875,
      "rewards/rejected": 1.046875,
      "step": 182
    },
    {
      "epoch": 0.732,
      "grad_norm": 52.267590702636376,
      "learning_rate": 1.7750257556651805e-07,
      "logits/chosen": -1.078125,
      "logits/rejected": -1.078125,
      "logps/chosen": -4.09375,
      "logps/rejected": -31.0,
      "loss": 1.1124,
      "nll_loss": 0.384765625,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.796875,
      "rewards/margins": 2.421875,
      "rewards/rejected": 0.375,
      "step": 183
    },
    {
      "epoch": 0.736,
      "grad_norm": 29.867593385467835,
      "learning_rate": 1.725696330273575e-07,
      "logits/chosen": -1.0390625,
      "logits/rejected": -1.0390625,
      "logps/chosen": -3.9375,
      "logps/rejected": -27.625,
      "loss": 1.0221,
      "nll_loss": 0.41015625,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 3.09375,
      "rewards/margins": 2.203125,
      "rewards/rejected": 0.890625,
      "step": 184
    },
    {
      "epoch": 0.74,
      "grad_norm": 25.170290330540713,
      "learning_rate": 1.6769187051016932e-07,
      "logits/chosen": -1.0703125,
      "logits/rejected": -1.0703125,
      "logps/chosen": -3.90625,
      "logps/rejected": -24.75,
      "loss": 1.0845,
      "nll_loss": 0.400390625,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.96875,
      "rewards/margins": 2.046875,
      "rewards/rejected": 0.90625,
      "step": 185
    },
    {
      "epoch": 0.744,
      "grad_norm": 17.759260894082026,
      "learning_rate": 1.6287011003719102e-07,
      "logits/chosen": -1.1015625,
      "logits/rejected": -1.109375,
      "logps/chosen": -4.5,
      "logps/rejected": -26.75,
      "loss": 1.0185,
      "nll_loss": 0.455078125,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 3.234375,
      "rewards/margins": 2.640625,
      "rewards/rejected": 0.6015625,
      "step": 186
    },
    {
      "epoch": 0.748,
      "grad_norm": 33.46964477076418,
      "learning_rate": 1.581051641929465e-07,
      "logits/chosen": -1.1015625,
      "logits/rejected": -1.0703125,
      "logps/chosen": -5.40625,
      "logps/rejected": -22.875,
      "loss": 1.1376,
      "nll_loss": 0.51953125,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.5625,
      "rewards/margins": 1.5,
      "rewards/rejected": 1.0625,
      "step": 187
    },
    {
      "epoch": 0.752,
      "grad_norm": 24.562745533923515,
      "learning_rate": 1.5339783598730565e-07,
      "logits/chosen": -1.046875,
      "logits/rejected": -1.046875,
      "logps/chosen": -5.5,
      "logps/rejected": -27.0,
      "loss": 1.1846,
      "nll_loss": 0.52734375,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 2.640625,
      "rewards/margins": 1.90625,
      "rewards/rejected": 0.73828125,
      "step": 188
    },
    {
      "epoch": 0.756,
      "grad_norm": 41.7355263163881,
      "learning_rate": 1.4874891872015734e-07,
      "logits/chosen": -1.046875,
      "logits/rejected": -1.046875,
      "logps/chosen": -5.90625,
      "logps/rejected": -25.25,
      "loss": 1.0575,
      "nll_loss": 0.546875,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.96875,
      "rewards/margins": 2.078125,
      "rewards/rejected": 0.89453125,
      "step": 189
    },
    {
      "epoch": 0.76,
      "grad_norm": 28.3791093205466,
      "learning_rate": 1.4415919584771995e-07,
      "logits/chosen": -1.0625,
      "logits/rejected": -1.0859375,
      "logps/chosen": -4.90625,
      "logps/rejected": -25.75,
      "loss": 1.1332,
      "nll_loss": 0.48828125,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 2.40625,
      "rewards/margins": 1.484375,
      "rewards/rejected": 0.92578125,
      "step": 190
    },
    {
      "epoch": 0.764,
      "grad_norm": 19.154245446560328,
      "learning_rate": 1.3962944085050832e-07,
      "logits/chosen": -1.0234375,
      "logits/rejected": -1.0390625,
      "logps/chosen": -6.125,
      "logps/rejected": -25.125,
      "loss": 1.0726,
      "nll_loss": 0.55078125,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.859375,
      "rewards/margins": 1.921875,
      "rewards/rejected": 0.93359375,
      "step": 191
    },
    {
      "epoch": 0.768,
      "grad_norm": 17.938238931793443,
      "learning_rate": 1.3516041710298497e-07,
      "logits/chosen": -1.0859375,
      "logits/rejected": -1.0703125,
      "logps/chosen": -4.125,
      "logps/rejected": -19.875,
      "loss": 0.9941,
      "nll_loss": 0.423828125,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.890625,
      "rewards/margins": 1.6015625,
      "rewards/rejected": 1.2890625,
      "step": 192
    },
    {
      "epoch": 0.772,
      "grad_norm": 20.81847675232155,
      "learning_rate": 1.3075287774491146e-07,
      "logits/chosen": -1.0703125,
      "logits/rejected": -1.1015625,
      "logps/chosen": -2.65625,
      "logps/rejected": -28.625,
      "loss": 1.0433,
      "nll_loss": 0.271484375,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 3.046875,
      "rewards/margins": 2.203125,
      "rewards/rejected": 0.83203125,
      "step": 193
    },
    {
      "epoch": 0.776,
      "grad_norm": 44.286085577032274,
      "learning_rate": 1.2640756555442683e-07,
      "logits/chosen": -1.0703125,
      "logits/rejected": -1.0546875,
      "logps/chosen": -6.5625,
      "logps/rejected": -25.125,
      "loss": 1.0916,
      "nll_loss": 0.64453125,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 3.078125,
      "rewards/margins": 1.9140625,
      "rewards/rejected": 1.1640625,
      "step": 194
    },
    {
      "epoch": 0.78,
      "grad_norm": 22.029622182069097,
      "learning_rate": 1.2212521282287093e-07,
      "logits/chosen": -1.0625,
      "logits/rejected": -1.0625,
      "logps/chosen": -4.5625,
      "logps/rejected": -28.125,
      "loss": 1.0946,
      "nll_loss": 0.462890625,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 2.9375,
      "rewards/margins": 2.0625,
      "rewards/rejected": 0.87109375,
      "step": 195
    },
    {
      "epoch": 0.784,
      "grad_norm": 11.861056714322848,
      "learning_rate": 1.179065412313755e-07,
      "logits/chosen": -1.1171875,
      "logits/rejected": -1.1015625,
      "logps/chosen": -6.5,
      "logps/rejected": -21.75,
      "loss": 1.0592,
      "nll_loss": 0.640625,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 2.90625,
      "rewards/margins": 1.6796875,
      "rewards/rejected": 1.2265625,
      "step": 196
    },
    {
      "epoch": 0.788,
      "grad_norm": 14.546735669018146,
      "learning_rate": 1.137522617292434e-07,
      "logits/chosen": -1.03125,
      "logits/rejected": -1.0234375,
      "logps/chosen": -4.34375,
      "logps/rejected": -18.5,
      "loss": 1.0507,
      "nll_loss": 0.41796875,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 3.109375,
      "rewards/margins": 1.3203125,
      "rewards/rejected": 1.7890625,
      "step": 197
    },
    {
      "epoch": 0.792,
      "grad_norm": 55.63915697780125,
      "learning_rate": 1.0966307441413597e-07,
      "logits/chosen": -1.1015625,
      "logits/rejected": -1.1015625,
      "logps/chosen": -6.125,
      "logps/rejected": -23.25,
      "loss": 1.0632,
      "nll_loss": 0.60546875,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 3.140625,
      "rewards/margins": 1.9921875,
      "rewards/rejected": 1.140625,
      "step": 198
    },
    {
      "epoch": 0.796,
      "grad_norm": 20.1761931612358,
      "learning_rate": 1.0563966841408995e-07,
      "logits/chosen": -1.0703125,
      "logits/rejected": -1.046875,
      "logps/chosen": -5.03125,
      "logps/rejected": -29.0,
      "loss": 0.9663,
      "nll_loss": 0.494140625,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 3.25,
      "rewards/margins": 2.546875,
      "rewards/rejected": 0.703125,
      "step": 199
    },
    {
      "epoch": 0.8,
      "grad_norm": 23.608862708071296,
      "learning_rate": 1.0168272177138199e-07,
      "logits/chosen": -1.0625,
      "logits/rejected": -1.0546875,
      "logps/chosen": -4.9375,
      "logps/rejected": -28.5,
      "loss": 1.0909,
      "nll_loss": 0.466796875,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 2.90625,
      "rewards/margins": 2.109375,
      "rewards/rejected": 0.8046875,
      "step": 200
    },
    {
      "epoch": 0.804,
      "grad_norm": 41.55024587771104,
      "learning_rate": 9.779290132826224e-08,
      "logits/chosen": -1.09375,
      "logits/rejected": -1.0859375,
      "logps/chosen": -4.84375,
      "logps/rejected": -24.125,
      "loss": 1.0829,
      "nll_loss": 0.44140625,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 3.390625,
      "rewards/margins": 1.9453125,
      "rewards/rejected": 1.4453125,
      "step": 201
    },
    {
      "epoch": 0.808,
      "grad_norm": 34.54633889712777,
      "learning_rate": 9.39708626145751e-08,
      "logits/chosen": -1.0546875,
      "logits/rejected": -1.046875,
      "logps/chosen": -5.90625,
      "logps/rejected": -27.25,
      "loss": 1.0733,
      "nll_loss": 0.57421875,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 3.28125,
      "rewards/margins": 2.203125,
      "rewards/rejected": 1.078125,
      "step": 202
    },
    {
      "epoch": 0.812,
      "grad_norm": 32.77492433560517,
      "learning_rate": 9.021724973728634e-08,
      "logits/chosen": -1.0546875,
      "logits/rejected": -1.078125,
      "logps/chosen": -4.875,
      "logps/rejected": -31.75,
      "loss": 1.0815,
      "nll_loss": 0.478515625,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.921875,
      "rewards/margins": 2.59375,
      "rewards/rejected": 0.318359375,
      "step": 203
    },
    {
      "epoch": 0.816,
      "grad_norm": 12.483799623280103,
      "learning_rate": 8.653269527193568e-08,
      "logits/chosen": -1.109375,
      "logits/rejected": -1.109375,
      "logps/chosen": -5.25,
      "logps/rejected": -25.375,
      "loss": 1.0666,
      "nll_loss": 0.486328125,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 3.0,
      "rewards/margins": 1.9140625,
      "rewards/rejected": 1.0859375,
      "step": 204
    },
    {
      "epoch": 0.82,
      "grad_norm": 52.55865885582505,
      "learning_rate": 8.291782015603176e-08,
      "logits/chosen": -1.140625,
      "logits/rejected": -1.09375,
      "logps/chosen": -6.0625,
      "logps/rejected": -20.375,
      "loss": 1.1337,
      "nll_loss": 0.63671875,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 2.625,
      "rewards/margins": 1.2578125,
      "rewards/rejected": 1.359375,
      "step": 205
    },
    {
      "epoch": 0.824,
      "grad_norm": 32.755044229250124,
      "learning_rate": 7.937323358440934e-08,
      "logits/chosen": -1.0703125,
      "logits/rejected": -1.0859375,
      "logps/chosen": -4.0625,
      "logps/rejected": -22.875,
      "loss": 1.0425,
      "nll_loss": 0.404296875,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 3.203125,
      "rewards/margins": 1.7109375,
      "rewards/rejected": 1.484375,
      "step": 206
    },
    {
      "epoch": 0.828,
      "grad_norm": 49.57046953822534,
      "learning_rate": 7.589953290656532e-08,
      "logits/chosen": -1.1171875,
      "logits/rejected": -1.1015625,
      "logps/chosen": -3.234375,
      "logps/rejected": -28.25,
      "loss": 1.1414,
      "nll_loss": 0.32421875,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 2.859375,
      "rewards/margins": 2.4375,
      "rewards/rejected": 0.435546875,
      "step": 207
    },
    {
      "epoch": 0.832,
      "grad_norm": 29.02033137242745,
      "learning_rate": 7.249730352598998e-08,
      "logits/chosen": -1.09375,
      "logits/rejected": -1.0859375,
      "logps/chosen": -4.8125,
      "logps/rejected": -28.75,
      "loss": 1.004,
      "nll_loss": 0.451171875,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.671875,
      "rewards/margins": 2.171875,
      "rewards/rejected": 0.51171875,
      "step": 208
    },
    {
      "epoch": 0.836,
      "grad_norm": 17.714740130325648,
      "learning_rate": 6.916711880151304e-08,
      "logits/chosen": -1.109375,
      "logits/rejected": -1.1015625,
      "logps/chosen": -6.25,
      "logps/rejected": -30.125,
      "loss": 1.1389,
      "nll_loss": 0.609375,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 2.703125,
      "rewards/margins": 2.109375,
      "rewards/rejected": 0.58984375,
      "step": 209
    },
    {
      "epoch": 0.84,
      "grad_norm": 17.72865807824904,
      "learning_rate": 6.59095399506781e-08,
      "logits/chosen": -1.1015625,
      "logits/rejected": -1.1015625,
      "logps/chosen": -6.5625,
      "logps/rejected": -26.5,
      "loss": 1.1747,
      "nll_loss": 0.6484375,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.8125,
      "rewards/margins": 1.875,
      "rewards/rejected": 0.94140625,
      "step": 210
    },
    {
      "epoch": 0.844,
      "grad_norm": 13.99041401845889,
      "learning_rate": 6.2725115955164e-08,
      "logits/chosen": -1.09375,
      "logits/rejected": -1.0859375,
      "logps/chosen": -4.1875,
      "logps/rejected": -33.5,
      "loss": 1.0243,
      "nll_loss": 0.3984375,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 2.96875,
      "rewards/margins": 2.828125,
      "rewards/rejected": 0.140625,
      "step": 211
    },
    {
      "epoch": 0.848,
      "grad_norm": 15.906985339916742,
      "learning_rate": 5.961438346826792e-08,
      "logits/chosen": -1.1015625,
      "logits/rejected": -1.0703125,
      "logps/chosen": -4.625,
      "logps/rejected": -28.0,
      "loss": 1.0347,
      "nll_loss": 0.443359375,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.59375,
      "rewards/margins": 2.03125,
      "rewards/rejected": 0.55859375,
      "step": 212
    },
    {
      "epoch": 0.852,
      "grad_norm": 36.27762496758833,
      "learning_rate": 5.6577866724466296e-08,
      "logits/chosen": -1.140625,
      "logits/rejected": -1.1171875,
      "logps/chosen": -6.53125,
      "logps/rejected": -47.75,
      "loss": 1.0298,
      "nll_loss": 0.58203125,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 2.328125,
      "rewards/margins": 3.59375,
      "rewards/rejected": -1.265625,
      "step": 213
    },
    {
      "epoch": 0.856,
      "grad_norm": 22.188902025969416,
      "learning_rate": 5.361607745106816e-08,
      "logits/chosen": -1.078125,
      "logits/rejected": -1.0703125,
      "logps/chosen": -3.734375,
      "logps/rejected": -22.5,
      "loss": 1.0263,
      "nll_loss": 0.431640625,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 3.0,
      "rewards/margins": 1.9375,
      "rewards/rejected": 1.0625,
      "step": 214
    },
    {
      "epoch": 0.86,
      "grad_norm": 26.514396481742928,
      "learning_rate": 5.072951478197723e-08,
      "logits/chosen": -1.1015625,
      "logits/rejected": -1.0703125,
      "logps/chosen": -4.46875,
      "logps/rejected": -31.5,
      "loss": 1.0853,
      "nll_loss": 0.4375,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 2.8125,
      "rewards/margins": 2.296875,
      "rewards/rejected": 0.515625,
      "step": 215
    },
    {
      "epoch": 0.864,
      "grad_norm": 39.18691508367433,
      "learning_rate": 4.79186651735749e-08,
      "logits/chosen": -1.0703125,
      "logits/rejected": -1.078125,
      "logps/chosen": -4.25,
      "logps/rejected": -23.625,
      "loss": 0.9658,
      "nll_loss": 0.427734375,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 2.828125,
      "rewards/margins": 1.625,
      "rewards/rejected": 1.2109375,
      "step": 216
    },
    {
      "epoch": 0.868,
      "grad_norm": 17.053221125940922,
      "learning_rate": 4.518400232274078e-08,
      "logits/chosen": -1.1171875,
      "logits/rejected": -1.109375,
      "logps/chosen": -4.90625,
      "logps/rejected": -27.125,
      "loss": 1.0356,
      "nll_loss": 0.4375,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 2.65625,
      "rewards/margins": 1.9140625,
      "rewards/rejected": 0.7421875,
      "step": 217
    },
    {
      "epoch": 0.872,
      "grad_norm": 30.04204392176379,
      "learning_rate": 4.252598708702343e-08,
      "logits/chosen": -1.0625,
      "logits/rejected": -1.0546875,
      "logps/chosen": -5.375,
      "logps/rejected": -24.25,
      "loss": 1.083,
      "nll_loss": 0.53125,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.765625,
      "rewards/margins": 1.5546875,
      "rewards/rejected": 1.203125,
      "step": 218
    },
    {
      "epoch": 0.876,
      "grad_norm": 12.813171720804812,
      "learning_rate": 3.994506740697406e-08,
      "logits/chosen": -1.046875,
      "logits/rejected": -1.0546875,
      "logps/chosen": -4.5625,
      "logps/rejected": -23.0,
      "loss": 0.9825,
      "nll_loss": 0.419921875,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.734375,
      "rewards/margins": 1.578125,
      "rewards/rejected": 1.15625,
      "step": 219
    },
    {
      "epoch": 0.88,
      "grad_norm": 31.189313546893203,
      "learning_rate": 3.744167823065814e-08,
      "logits/chosen": -1.0859375,
      "logits/rejected": -1.078125,
      "logps/chosen": -4.96875,
      "logps/rejected": -36.0,
      "loss": 1.0805,
      "nll_loss": 0.478515625,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.296875,
      "rewards/margins": 2.71875,
      "rewards/rejected": -0.4296875,
      "step": 220
    },
    {
      "epoch": 0.884,
      "grad_norm": 38.49107579612584,
      "learning_rate": 3.501624144035559e-08,
      "logits/chosen": -1.1015625,
      "logits/rejected": -1.0703125,
      "logps/chosen": -5.5,
      "logps/rejected": -23.375,
      "loss": 1.097,
      "nll_loss": 0.53125,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.78125,
      "rewards/margins": 1.6953125,
      "rewards/rejected": 1.09375,
      "step": 221
    },
    {
      "epoch": 0.888,
      "grad_norm": 13.5024131062282,
      "learning_rate": 3.2669165781463395e-08,
      "logits/chosen": -1.0859375,
      "logits/rejected": -1.078125,
      "logps/chosen": -4.53125,
      "logps/rejected": -29.875,
      "loss": 1.0892,
      "nll_loss": 0.44921875,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.765625,
      "rewards/margins": 1.625,
      "rewards/rejected": 1.140625,
      "step": 222
    },
    {
      "epoch": 0.892,
      "grad_norm": 28.282704129587508,
      "learning_rate": 3.0400846793612544e-08,
      "logits/chosen": -1.0859375,
      "logits/rejected": -1.0546875,
      "logps/chosen": -4.71875,
      "logps/rejected": -23.25,
      "loss": 1.0451,
      "nll_loss": 0.462890625,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.859375,
      "rewards/margins": 1.28125,
      "rewards/rejected": 1.578125,
      "step": 223
    },
    {
      "epoch": 0.896,
      "grad_norm": 37.60742006233887,
      "learning_rate": 2.8211666744009045e-08,
      "logits/chosen": -1.1328125,
      "logits/rejected": -1.125,
      "logps/chosen": -3.796875,
      "logps/rejected": -29.875,
      "loss": 1.0323,
      "nll_loss": 0.333984375,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.671875,
      "rewards/margins": 2.265625,
      "rewards/rejected": 0.41015625,
      "step": 224
    },
    {
      "epoch": 0.9,
      "grad_norm": 29.516319250938352,
      "learning_rate": 2.6101994563013098e-08,
      "logits/chosen": -1.078125,
      "logits/rejected": -1.09375,
      "logps/chosen": -3.453125,
      "logps/rejected": -26.25,
      "loss": 1.0479,
      "nll_loss": 0.341796875,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 2.59375,
      "rewards/margins": 1.796875,
      "rewards/rejected": 0.7890625,
      "step": 225
    },
    {
      "epoch": 0.904,
      "grad_norm": 14.847458095289273,
      "learning_rate": 2.4072185781965238e-08,
      "logits/chosen": -1.0859375,
      "logits/rejected": -1.09375,
      "logps/chosen": -4.8125,
      "logps/rejected": -29.25,
      "loss": 1.0546,
      "nll_loss": 0.43359375,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 2.9375,
      "rewards/margins": 2.203125,
      "rewards/rejected": 0.7265625,
      "step": 226
    },
    {
      "epoch": 0.908,
      "grad_norm": 17.36079249416162,
      "learning_rate": 2.212258247327059e-08,
      "logits/chosen": -1.109375,
      "logits/rejected": -1.09375,
      "logps/chosen": -3.578125,
      "logps/rejected": -31.875,
      "loss": 1.0537,
      "nll_loss": 0.359375,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.765625,
      "rewards/margins": 2.34375,
      "rewards/rejected": 0.4375,
      "step": 227
    },
    {
      "epoch": 0.912,
      "grad_norm": 18.768525578098558,
      "learning_rate": 2.025351319275137e-08,
      "logits/chosen": -1.1015625,
      "logits/rejected": -1.0859375,
      "logps/chosen": -6.59375,
      "logps/rejected": -38.25,
      "loss": 1.106,
      "nll_loss": 0.56640625,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.875,
      "rewards/margins": 3.203125,
      "rewards/rejected": -0.31640625,
      "step": 228
    },
    {
      "epoch": 0.916,
      "grad_norm": 33.5917307298862,
      "learning_rate": 1.8465292924276844e-08,
      "logits/chosen": -1.1484375,
      "logits/rejected": -1.1484375,
      "logps/chosen": -5.09375,
      "logps/rejected": -42.75,
      "loss": 0.9739,
      "nll_loss": 0.46875,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 3.078125,
      "rewards/margins": 3.578125,
      "rewards/rejected": -0.50390625,
      "step": 229
    },
    {
      "epoch": 0.92,
      "grad_norm": 12.955837772485086,
      "learning_rate": 1.6758223026681507e-08,
      "logits/chosen": -1.0703125,
      "logits/rejected": -1.078125,
      "logps/chosen": -6.1875,
      "logps/rejected": -30.75,
      "loss": 1.0963,
      "nll_loss": 0.58203125,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.734375,
      "rewards/margins": 2.234375,
      "rewards/rejected": 0.5,
      "step": 230
    },
    {
      "epoch": 0.924,
      "grad_norm": 16.2992963110193,
      "learning_rate": 1.5132591182978105e-08,
      "logits/chosen": -1.0625,
      "logits/rejected": -1.0546875,
      "logps/chosen": -6.28125,
      "logps/rejected": -31.875,
      "loss": 1.0419,
      "nll_loss": 0.56640625,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 2.75,
      "rewards/margins": 2.265625,
      "rewards/rejected": 0.482421875,
      "step": 231
    },
    {
      "epoch": 0.928,
      "grad_norm": 12.293312024242535,
      "learning_rate": 1.3588671351876357e-08,
      "logits/chosen": -1.0703125,
      "logits/rejected": -1.0859375,
      "logps/chosen": -4.53125,
      "logps/rejected": -22.875,
      "loss": 1.0256,
      "nll_loss": 0.439453125,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 2.59375,
      "rewards/margins": 1.4375,
      "rewards/rejected": 1.15625,
      "step": 232
    },
    {
      "epoch": 0.932,
      "grad_norm": 18.415969639868365,
      "learning_rate": 1.2126723721614052e-08,
      "logits/chosen": -1.0859375,
      "logits/rejected": -1.109375,
      "logps/chosen": -4.71875,
      "logps/rejected": -28.5,
      "loss": 1.0731,
      "nll_loss": 0.46484375,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.59375,
      "rewards/margins": 2.203125,
      "rewards/rejected": 0.376953125,
      "step": 233
    },
    {
      "epoch": 0.936,
      "grad_norm": 17.63813786850423,
      "learning_rate": 1.0746994666109233e-08,
      "logits/chosen": -1.109375,
      "logits/rejected": -1.0859375,
      "logps/chosen": -4.75,
      "logps/rejected": -20.5,
      "loss": 0.9922,
      "nll_loss": 0.431640625,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 3.46875,
      "rewards/margins": 1.765625,
      "rewards/rejected": 1.6953125,
      "step": 234
    },
    {
      "epoch": 0.94,
      "grad_norm": 15.498735501179883,
      "learning_rate": 9.449716703439803e-09,
      "logits/chosen": -1.1171875,
      "logits/rejected": -1.140625,
      "logps/chosen": -4.6875,
      "logps/rejected": -23.625,
      "loss": 1.0851,
      "nll_loss": 0.404296875,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.9375,
      "rewards/margins": 2.046875,
      "rewards/rejected": 0.89453125,
      "step": 235
    },
    {
      "epoch": 0.944,
      "grad_norm": 11.978784225505468,
      "learning_rate": 8.235108456658812e-09,
      "logits/chosen": -1.140625,
      "logits/rejected": -1.1328125,
      "logps/chosen": -4.03125,
      "logps/rejected": -39.25,
      "loss": 1.0219,
      "nll_loss": 0.390625,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 2.59375,
      "rewards/margins": 3.0,
      "rewards/rejected": -0.41015625,
      "step": 236
    },
    {
      "epoch": 0.948,
      "grad_norm": 13.770414732547914,
      "learning_rate": 7.103374616951041e-09,
      "logits/chosen": -1.0546875,
      "logits/rejected": -1.0546875,
      "logps/chosen": -3.25,
      "logps/rejected": -21.75,
      "loss": 1.0617,
      "nll_loss": 0.322265625,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 3.3125,
      "rewards/margins": 1.7109375,
      "rewards/rejected": 1.6015625,
      "step": 237
    },
    {
      "epoch": 0.952,
      "grad_norm": 26.345272412615557,
      "learning_rate": 6.054705909137425e-09,
      "logits/chosen": -1.1015625,
      "logits/rejected": -1.125,
      "logps/chosen": -4.1875,
      "logps/rejected": -38.0,
      "loss": 1.078,
      "nll_loss": 0.443359375,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.734375,
      "rewards/margins": 3.03125,
      "rewards/rejected": -0.30078125,
      "step": 238
    },
    {
      "epoch": 0.956,
      "grad_norm": 17.86401941431699,
      "learning_rate": 5.0892790595336575e-09,
      "logits/chosen": -1.125,
      "logits/rejected": -1.1328125,
      "logps/chosen": -4.4375,
      "logps/rejected": -27.5,
      "loss": 0.9794,
      "nll_loss": 0.451171875,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 2.640625,
      "rewards/margins": 2.140625,
      "rewards/rejected": 0.49609375,
      "step": 239
    },
    {
      "epoch": 0.96,
      "grad_norm": 17.07158211116171,
      "learning_rate": 4.207256766166845e-09,
      "logits/chosen": -1.078125,
      "logits/rejected": -1.0859375,
      "logps/chosen": -4.90625,
      "logps/rejected": -26.625,
      "loss": 1.1286,
      "nll_loss": 0.443359375,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.453125,
      "rewards/margins": 1.578125,
      "rewards/rejected": 0.875,
      "step": 240
    },
    {
      "epoch": 0.964,
      "grad_norm": 40.38091842339886,
      "learning_rate": 3.4087876713574935e-09,
      "logits/chosen": -1.1015625,
      "logits/rejected": -1.09375,
      "logps/chosen": -5.96875,
      "logps/rejected": -31.375,
      "loss": 0.9722,
      "nll_loss": 0.609375,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 3.015625,
      "rewards/margins": 2.421875,
      "rewards/rejected": 0.58984375,
      "step": 241
    },
    {
      "epoch": 0.968,
      "grad_norm": 14.247494421593556,
      "learning_rate": 2.69400633666933e-09,
      "logits/chosen": -1.1171875,
      "logits/rejected": -1.109375,
      "logps/chosen": -4.4375,
      "logps/rejected": -21.375,
      "loss": 1.0203,
      "nll_loss": 0.408203125,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.71875,
      "rewards/margins": 1.640625,
      "rewards/rejected": 1.078125,
      "step": 242
    },
    {
      "epoch": 0.972,
      "grad_norm": 30.909416470072028,
      "learning_rate": 2.0630332202324906e-09,
      "logits/chosen": -1.0546875,
      "logits/rejected": -1.0859375,
      "logps/chosen": -5.65625,
      "logps/rejected": -34.25,
      "loss": 1.0594,
      "nll_loss": 0.5,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 3.03125,
      "rewards/margins": 2.96875,
      "rewards/rejected": 0.06396484375,
      "step": 243
    },
    {
      "epoch": 0.976,
      "grad_norm": 13.79697350875317,
      "learning_rate": 1.5159746564433196e-09,
      "logits/chosen": -1.09375,
      "logits/rejected": -1.09375,
      "logps/chosen": -5.875,
      "logps/rejected": -31.625,
      "loss": 1.0882,
      "nll_loss": 0.59375,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.625,
      "rewards/margins": 2.5,
      "rewards/rejected": 0.1337890625,
      "step": 244
    },
    {
      "epoch": 0.98,
      "grad_norm": 26.292855916544468,
      "learning_rate": 1.0529228380445899e-09,
      "logits/chosen": -1.125,
      "logits/rejected": -1.1015625,
      "logps/chosen": -4.9375,
      "logps/rejected": -36.0,
      "loss": 1.0991,
      "nll_loss": 0.4453125,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 2.921875,
      "rewards/margins": 2.765625,
      "rewards/rejected": 0.1630859375,
      "step": 245
    },
    {
      "epoch": 0.984,
      "grad_norm": 9.993888997764955,
      "learning_rate": 6.739558005884883e-10,
      "logits/chosen": -1.0625,
      "logits/rejected": -1.0546875,
      "logps/chosen": -6.125,
      "logps/rejected": -25.625,
      "loss": 1.0262,
      "nll_loss": 0.52734375,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 2.78125,
      "rewards/margins": 1.671875,
      "rewards/rejected": 1.109375,
      "step": 246
    },
    {
      "epoch": 0.988,
      "grad_norm": 26.071889768691495,
      "learning_rate": 3.791374092859678e-10,
      "logits/chosen": -1.1171875,
      "logits/rejected": -1.1328125,
      "logps/chosen": -6.34375,
      "logps/rejected": -31.75,
      "loss": 1.0238,
      "nll_loss": 0.55078125,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.515625,
      "rewards/margins": 2.046875,
      "rewards/rejected": 0.46875,
      "step": 247
    },
    {
      "epoch": 0.992,
      "grad_norm": 27.845797479287505,
      "learning_rate": 1.6851734824380182e-10,
      "logits/chosen": -1.0703125,
      "logits/rejected": -1.078125,
      "logps/chosen": -3.578125,
      "logps/rejected": -32.75,
      "loss": 1.1748,
      "nll_loss": 0.341796875,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.953125,
      "rewards/margins": 2.265625,
      "rewards/rejected": 0.6875,
      "step": 248
    },
    {
      "epoch": 0.996,
      "grad_norm": 16.83674597083853,
      "learning_rate": 4.213111209155906e-11,
      "logits/chosen": -1.140625,
      "logits/rejected": -1.125,
      "logps/chosen": -4.1875,
      "logps/rejected": -30.125,
      "loss": 0.9346,
      "nll_loss": 0.38671875,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 2.984375,
      "rewards/margins": 2.625,
      "rewards/rejected": 0.35546875,
      "step": 249
    },
    {
      "epoch": 1.0,
      "grad_norm": 24.975249728436633,
      "learning_rate": 0.0,
      "logits/chosen": -1.09375,
      "logits/rejected": -1.0859375,
      "logps/chosen": -5.1875,
      "logps/rejected": -30.875,
      "loss": 1.0326,
      "nll_loss": 0.5078125,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.375,
      "rewards/margins": 2.171875,
      "rewards/rejected": 0.2060546875,
      "step": 250
    },
    {
      "epoch": 1.0,
      "step": 250,
      "total_flos": 1.517913978562874e+20,
      "train_loss": 1.4306324501037597,
      "train_runtime": 16604.645,
      "train_samples_per_second": 3.854,
      "train_steps_per_second": 0.015
    }
  ],
  "logging_steps": 1.0,
  "max_steps": 250,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 1.517913978562874e+20,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}