{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 50.0,
  "eval_steps": 500,
  "global_step": 250,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "completion_length": 500.0,
      "epoch": 0.2,
      "grad_norm": 144.23707580566406,
      "kl": 51.48179244995117,
      "learning_rate": 3.1250000000000005e-08,
      "loss": 2.0593,
      "reward": 0.9761996865272522,
      "reward_std": 3.3251326084136963,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.148800253868103,
      "rewards/wrapped_format_reward": 0.625,
      "step": 1
    },
    {
      "completion_length": 500.0,
      "epoch": 0.4,
      "grad_norm": 974.2139892578125,
      "kl": 216.24957275390625,
      "learning_rate": 6.250000000000001e-08,
      "loss": 8.65,
      "reward": -3.75,
      "reward_std": 0.5,
      "rewards/mpc_param_extraction_reward": 0.0,
      "rewards/mpc_param_name_reward": 0.0,
      "rewards/wrapped_driving_reward": -4.0,
      "rewards/wrapped_format_reward": 0.25,
      "step": 2
    },
    {
      "completion_length": 500.0,
      "epoch": 0.6,
      "grad_norm": 41697.91015625,
      "kl": 3837.41943359375,
      "learning_rate": 9.375e-08,
      "loss": 153.4967,
      "reward": -0.7961921691894531,
      "reward_std": 3.700653076171875,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.4375,
      "rewards/wrapped_driving_reward": -1.7336921691894531,
      "rewards/wrapped_format_reward": 0.0,
      "step": 3
    },
    {
      "completion_length": 500.0,
      "epoch": 0.8,
      "grad_norm": 10122959.0,
      "kl": 511094.90625,
      "learning_rate": 1.2500000000000002e-07,
      "loss": 20443.7988,
      "reward": -2.338921546936035,
      "reward_std": 3.322157144546509,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -2.838921546936035,
      "rewards/wrapped_format_reward": 0.0,
      "step": 4
    },
    {
      "completion_length": 500.0,
      "epoch": 1.0,
      "grad_norm": 100702232.0,
      "kl": 5416315.0,
      "learning_rate": 1.5625e-07,
      "loss": 216652.5938,
      "reward": -0.16450506448745728,
      "reward_std": 3.8515079021453857,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.6645050048828125,
      "rewards/wrapped_format_reward": 0.5,
      "step": 5
    },
    {
      "completion_length": 500.0,
      "epoch": 1.2,
      "grad_norm": 17.33243751525879,
      "kl": 7.672175884246826,
      "learning_rate": 1.875e-07,
      "loss": 0.3069,
      "reward": 0.9893605709075928,
      "reward_std": 1.5257619619369507,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -1.3856394290924072,
      "rewards/wrapped_format_reward": 0.375,
      "step": 6
    },
    {
      "completion_length": 500.0,
      "epoch": 1.4,
      "grad_norm": 70.34513092041016,
      "kl": 17.917146682739258,
      "learning_rate": 2.1875e-07,
      "loss": 0.7167,
      "reward": 1.2267450094223022,
      "reward_std": 3.4932949542999268,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.5232549905776978,
      "rewards/wrapped_format_reward": 0.25,
      "step": 7
    },
    {
      "completion_length": 500.0,
      "epoch": 1.6,
      "grad_norm": 69448.7734375,
      "kl": 9786.2802734375,
      "learning_rate": 2.5000000000000004e-07,
      "loss": 391.4512,
      "reward": -0.967779815196991,
      "reward_std": 3.5180184841156006,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.2177798748016357,
      "rewards/wrapped_format_reward": 0.25,
      "step": 8
    },
    {
      "completion_length": 500.0,
      "epoch": 1.8,
      "grad_norm": 7205431.5,
      "kl": 363326.15625,
      "learning_rate": 2.8125e-07,
      "loss": 14533.0439,
      "reward": -0.4434952139854431,
      "reward_std": 4.112156867980957,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.5684951543807983,
      "rewards/wrapped_format_reward": 0.125,
      "step": 9
    },
    {
      "completion_length": 500.0,
      "epoch": 2.0,
      "grad_norm": 1344.39306640625,
      "kl": 182.57179260253906,
      "learning_rate": 3.125e-07,
      "loss": 7.3029,
      "reward": -0.5283111929893494,
      "reward_std": 3.7256903648376465,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.6533112525939941,
      "rewards/wrapped_format_reward": 0.125,
      "step": 10
    },
    {
      "completion_length": 500.0,
      "epoch": 2.2,
      "grad_norm": 663.510498046875,
      "kl": 125.65680694580078,
      "learning_rate": 3.4375000000000004e-07,
      "loss": 5.0263,
      "reward": -2.449397563934326,
      "reward_std": 3.1012051105499268,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -2.949397563934326,
      "rewards/wrapped_format_reward": 0.0,
      "step": 11
    },
    {
      "completion_length": 500.0,
      "epoch": 2.4,
      "grad_norm": 31.331226348876953,
      "kl": 10.755382537841797,
      "learning_rate": 3.75e-07,
      "loss": 0.4302,
      "reward": -2.1561226844787598,
      "reward_std": 3.36269211769104,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -2.7811226844787598,
      "rewards/wrapped_format_reward": 0.125,
      "step": 12
    },
    {
      "completion_length": 500.0,
      "epoch": 2.6,
      "grad_norm": 10.003079414367676,
      "kl": 3.8625946044921875,
      "learning_rate": 4.0625000000000003e-07,
      "loss": 0.1545,
      "reward": -2.1760454177856445,
      "reward_std": 3.647908926010132,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -2.8010454177856445,
      "rewards/wrapped_format_reward": 0.125,
      "step": 13
    },
    {
      "completion_length": 500.0,
      "epoch": 2.8,
      "grad_norm": 62.2325325012207,
      "kl": 13.510702133178711,
      "learning_rate": 4.375e-07,
      "loss": 0.5404,
      "reward": -2.294018030166626,
      "reward_std": 3.0876402854919434,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -3.044018030166626,
      "rewards/wrapped_format_reward": 0.25,
      "step": 14
    },
    {
      "completion_length": 500.0,
      "epoch": 3.0,
      "grad_norm": 96.1074447631836,
      "kl": 10.679292678833008,
      "learning_rate": 4.6875000000000006e-07,
      "loss": 0.4272,
      "reward": -0.6371059417724609,
      "reward_std": 3.885227680206299,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.637105941772461,
      "rewards/wrapped_format_reward": 0.0,
      "step": 15
    },
    {
      "completion_length": 500.0,
      "epoch": 3.2,
      "grad_norm": 1650.8782958984375,
      "kl": 208.3596954345703,
      "learning_rate": 5.000000000000001e-07,
      "loss": 8.3344,
      "reward": -4.0,
      "reward_std": 0.0,
      "rewards/mpc_param_extraction_reward": 0.0,
      "rewards/mpc_param_name_reward": 0.0,
      "rewards/wrapped_driving_reward": -4.0,
      "rewards/wrapped_format_reward": 0.0,
      "step": 16
    },
    {
      "completion_length": 500.0,
      "epoch": 3.4,
      "grad_norm": 17.093393325805664,
      "kl": 5.905396461486816,
      "learning_rate": 5.3125e-07,
      "loss": 0.2362,
      "reward": -2.4352118968963623,
      "reward_std": 2.806159257888794,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -3.060211658477783,
      "rewards/wrapped_format_reward": 0.125,
      "step": 17
    },
    {
      "completion_length": 500.0,
      "epoch": 3.6,
      "grad_norm": 78087.4140625,
      "kl": 7675.3564453125,
      "learning_rate": 5.625e-07,
      "loss": 307.0142,
      "reward": -0.4786604046821594,
      "reward_std": 4.071903228759766,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.8536603450775146,
      "rewards/wrapped_format_reward": 0.375,
      "step": 18
    },
    {
      "completion_length": 500.0,
      "epoch": 3.8,
      "grad_norm": 2062.067626953125,
      "kl": 105.56303405761719,
      "learning_rate": 5.9375e-07,
      "loss": 4.2225,
      "reward": -0.4283701777458191,
      "reward_std": 4.145442008972168,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.8033702373504639,
      "rewards/wrapped_format_reward": 0.375,
      "step": 19
    },
    {
      "completion_length": 500.0,
      "epoch": 4.0,
      "grad_norm": 513992.65625,
      "kl": 39077.08984375,
      "learning_rate": 6.25e-07,
      "loss": 1563.0836,
      "reward": -2.0327651500701904,
      "reward_std": 3.3016297817230225,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -2.7827651500701904,
      "rewards/wrapped_format_reward": 0.25,
      "step": 20
    },
    {
      "completion_length": 500.0,
      "epoch": 4.2,
      "grad_norm": 271.5398254394531,
      "kl": 37.08869934082031,
      "learning_rate": 6.562500000000001e-07,
      "loss": 1.4835,
      "reward": -3.5,
      "reward_std": 0.5773502588272095,
      "rewards/mpc_param_extraction_reward": 0.0,
      "rewards/mpc_param_name_reward": 0.0,
      "rewards/wrapped_driving_reward": -4.0,
      "rewards/wrapped_format_reward": 0.5,
      "step": 21
    },
    {
      "completion_length": 500.0,
      "epoch": 4.4,
      "grad_norm": 33774.453125,
      "kl": 4115.1591796875,
      "learning_rate": 6.875000000000001e-07,
      "loss": 164.6064,
      "reward": -1.1355788707733154,
      "reward_std": 3.42315673828125,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.5105788707733154,
      "rewards/wrapped_format_reward": 0.375,
      "step": 22
    },
    {
      "completion_length": 500.0,
      "epoch": 4.6,
      "grad_norm": 52.09832000732422,
      "kl": 14.069100379943848,
      "learning_rate": 7.1875e-07,
      "loss": 0.5628,
      "reward": 0.9047523736953735,
      "reward_std": 3.2798702716827393,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.5952475666999817,
      "rewards/wrapped_format_reward": 0.0,
      "step": 23
    },
    {
      "completion_length": 500.0,
      "epoch": 4.8,
      "grad_norm": 75.83870697021484,
      "kl": 16.262989044189453,
      "learning_rate": 7.5e-07,
      "loss": 0.6505,
      "reward": -0.5572073459625244,
      "reward_std": 2.9710958003997803,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -2.0572073459625244,
      "rewards/wrapped_format_reward": 0.0,
      "step": 24
    },
    {
      "completion_length": 500.0,
      "epoch": 5.0,
      "grad_norm": 8.57257080078125,
      "kl": 3.3865182399749756,
      "learning_rate": 7.8125e-07,
      "loss": 0.1355,
      "reward": -2.110412120819092,
      "reward_std": 3.7791755199432373,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -2.860412120819092,
      "rewards/wrapped_format_reward": 0.25,
      "step": 25
    },
    {
      "completion_length": 500.0,
      "epoch": 5.2,
      "grad_norm": 978403.0,
      "kl": 89647.2578125,
      "learning_rate": 8.125000000000001e-07,
      "loss": 3585.8899,
      "reward": -0.2944529056549072,
      "reward_std": 4.2804718017578125,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.6694529056549072,
      "rewards/wrapped_format_reward": 0.375,
      "step": 26
    },
    {
      "completion_length": 500.0,
      "epoch": 5.4,
      "grad_norm": 168.73036193847656,
      "kl": 29.724079132080078,
      "learning_rate": 8.437500000000001e-07,
      "loss": 1.189,
      "reward": -0.6913368701934814,
      "reward_std": 3.5795211791992188,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.0663368701934814,
      "rewards/wrapped_format_reward": 0.375,
      "step": 27
    },
    {
      "completion_length": 500.0,
      "epoch": 5.6,
      "grad_norm": 75324.5234375,
      "kl": 7936.74267578125,
      "learning_rate": 8.75e-07,
      "loss": 317.4697,
      "reward": 0.9355948567390442,
      "reward_std": 3.3464736938476562,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.6894051432609558,
      "rewards/wrapped_format_reward": 0.125,
      "step": 28
    },
    {
      "completion_length": 500.0,
      "epoch": 5.8,
      "grad_norm": 469.9671630859375,
      "kl": 71.2878189086914,
      "learning_rate": 9.0625e-07,
      "loss": 2.8515,
      "reward": -2.547950267791748,
      "reward_std": 2.904099464416504,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -3.047950267791748,
      "rewards/wrapped_format_reward": 0.0,
      "step": 29
    },
    {
      "completion_length": 500.0,
      "epoch": 6.0,
      "grad_norm": 36.652645111083984,
      "kl": 13.148932456970215,
      "learning_rate": 9.375000000000001e-07,
      "loss": 0.526,
      "reward": -3.375,
      "reward_std": 0.9464846849441528,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -4.0,
      "rewards/wrapped_format_reward": 0.125,
      "step": 30
    },
    {
      "completion_length": 500.0,
      "epoch": 6.2,
      "grad_norm": 30.199684143066406,
      "kl": 9.480849266052246,
      "learning_rate": 9.6875e-07,
      "loss": 0.3792,
      "reward": -2.0530290603637695,
      "reward_std": 3.2615222930908203,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -2.9280290603637695,
      "rewards/wrapped_format_reward": 0.375,
      "step": 31
    },
    {
      "completion_length": 500.0,
      "epoch": 6.4,
      "grad_norm": 78.3298568725586,
      "kl": 26.2161865234375,
      "learning_rate": 1.0000000000000002e-06,
      "loss": 1.0486,
      "reward": 1.33430814743042,
      "reward_std": 3.5583572387695312,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.41569197177886963,
      "rewards/wrapped_format_reward": 0.25,
      "step": 32
    },
    {
      "completion_length": 500.0,
      "epoch": 6.6,
      "grad_norm": 19.472774505615234,
      "kl": 7.02009391784668,
      "learning_rate": 1.03125e-06,
      "loss": 0.2808,
      "reward": -0.6657888889312744,
      "reward_std": 3.853987216949463,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.6657888889312744,
      "rewards/wrapped_format_reward": 0.0,
      "step": 33
    },
    {
      "completion_length": 500.0,
      "epoch": 6.8,
      "grad_norm": 65.95396423339844,
      "kl": 18.14912223815918,
      "learning_rate": 1.0625e-06,
      "loss": 0.726,
      "reward": -0.8661626577377319,
      "reward_std": 3.6189870834350586,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.1161625385284424,
      "rewards/wrapped_format_reward": 0.25,
      "step": 34
    },
    {
      "completion_length": 500.0,
      "epoch": 7.0,
      "grad_norm": 1.4008393287658691,
      "kl": 0.8411699533462524,
      "learning_rate": 1.0937500000000001e-06,
      "loss": 0.0336,
      "reward": -0.38928359746932983,
      "reward_std": 4.171046257019043,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.6392836570739746,
      "rewards/wrapped_format_reward": 0.25,
      "step": 35
    },
    {
      "completion_length": 500.0,
      "epoch": 7.2,
      "grad_norm": 7.80991268157959,
      "kl": 4.218427658081055,
      "learning_rate": 1.125e-06,
      "loss": 0.1687,
      "reward": -1.1417465209960938,
      "reward_std": 3.381352186203003,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.1417465209960938,
      "rewards/wrapped_format_reward": 0.0,
      "step": 36
    },
    {
      "completion_length": 500.0,
      "epoch": 7.4,
      "grad_norm": 4.478097438812256,
      "kl": 2.112290143966675,
      "learning_rate": 1.1562500000000002e-06,
      "loss": 0.0845,
      "reward": -2.102426528930664,
      "reward_std": 3.1373467445373535,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -2.852426528930664,
      "rewards/wrapped_format_reward": 0.25,
      "step": 37
    },
    {
      "completion_length": 500.0,
      "epoch": 7.6,
      "grad_norm": 9.182758331298828,
      "kl": 4.63405179977417,
      "learning_rate": 1.1875e-06,
      "loss": 0.1854,
      "reward": -2.2051236629486084,
      "reward_std": 3.2649383544921875,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -2.9551236629486084,
      "rewards/wrapped_format_reward": 0.25,
      "step": 38
    },
    {
      "completion_length": 500.0,
      "epoch": 7.8,
      "grad_norm": 203072.515625,
      "kl": 20056.82421875,
      "learning_rate": 1.21875e-06,
      "loss": 802.2729,
      "reward": -2.125791311264038,
      "reward_std": 3.0907511711120605,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -3.000791311264038,
      "rewards/wrapped_format_reward": 0.375,
      "step": 39
    },
    {
      "completion_length": 500.0,
      "epoch": 8.0,
      "grad_norm": 5.843477725982666,
      "kl": 3.436691999435425,
      "learning_rate": 1.25e-06,
      "loss": 0.1375,
      "reward": -0.9351435899734497,
      "reward_std": 3.608586072921753,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.3101437091827393,
      "rewards/wrapped_format_reward": 0.375,
      "step": 40
    },
    {
      "completion_length": 500.0,
      "epoch": 8.2,
      "grad_norm": 7742784.5,
      "kl": 650933.375,
      "learning_rate": 1.28125e-06,
      "loss": 26037.334,
      "reward": 1.5518302917480469,
      "reward_std": 3.7290942668914795,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.4481697678565979,
      "rewards/wrapped_format_reward": 0.5,
      "step": 41
    },
    {
      "completion_length": 500.0,
      "epoch": 8.4,
      "grad_norm": 8.937725067138672,
      "kl": 3.0701639652252197,
      "learning_rate": 1.3125000000000001e-06,
      "loss": 0.1228,
      "reward": -2.1904397010803223,
      "reward_std": 3.2942306995391846,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -3.0654397010803223,
      "rewards/wrapped_format_reward": 0.375,
      "step": 42
    },
    {
      "completion_length": 500.0,
      "epoch": 8.6,
      "grad_norm": 14.36681079864502,
      "kl": 5.88793420791626,
      "learning_rate": 1.34375e-06,
      "loss": 0.2355,
      "reward": 0.6519123315811157,
      "reward_std": 3.105113983154297,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.973087728023529,
      "rewards/wrapped_format_reward": 0.125,
      "step": 43
    },
    {
      "completion_length": 500.0,
      "epoch": 8.8,
      "grad_norm": 180.1724395751953,
      "kl": 33.255760192871094,
      "learning_rate": 1.3750000000000002e-06,
      "loss": 1.3302,
      "reward": -0.6511552333831787,
      "reward_std": 3.9030916690826416,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.7761552333831787,
      "rewards/wrapped_format_reward": 0.125,
      "step": 44
    },
    {
      "completion_length": 500.0,
      "epoch": 9.0,
      "grad_norm": 32.81709671020508,
      "kl": 6.7791428565979,
      "learning_rate": 1.40625e-06,
      "loss": 0.2712,
      "reward": -3.875,
      "reward_std": 0.25,
      "rewards/mpc_param_extraction_reward": 0.0,
      "rewards/mpc_param_name_reward": 0.0,
      "rewards/wrapped_driving_reward": -4.0,
      "rewards/wrapped_format_reward": 0.125,
      "step": 45
    },
    {
      "completion_length": 500.0,
      "epoch": 9.2,
      "grad_norm": 4151.640625,
      "kl": 350.8572082519531,
      "learning_rate": 1.4375e-06,
      "loss": 14.0343,
      "reward": -1.1032943725585938,
      "reward_std": 3.3453028202056885,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.1032943725585938,
      "rewards/wrapped_format_reward": 0.0,
      "step": 46
    },
    {
      "completion_length": 500.0,
      "epoch": 9.4,
      "grad_norm": 15.409821510314941,
      "kl": 5.346187114715576,
      "learning_rate": 1.4687500000000001e-06,
      "loss": 0.2138,
      "reward": -1.1791430711746216,
      "reward_std": 2.8540005683898926,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -2.929143190383911,
      "rewards/wrapped_format_reward": 0.25,
      "step": 47
    },
    {
      "completion_length": 500.0,
      "epoch": 9.6,
      "grad_norm": 144.20443725585938,
      "kl": 23.608051300048828,
      "learning_rate": 1.5e-06,
      "loss": 0.9443,
      "reward": -2.530029058456421,
      "reward_std": 2.939941883087158,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.20000000298023224,
      "rewards/wrapped_driving_reward": -2.9800291061401367,
      "rewards/wrapped_format_reward": 0.0,
      "step": 48
    },
    {
      "completion_length": 500.0,
      "epoch": 9.8,
      "grad_norm": 7.547443866729736,
      "kl": 3.824962615966797,
      "learning_rate": 1.5312500000000002e-06,
      "loss": 0.153,
      "reward": -0.7816690802574158,
      "reward_std": 3.7201201915740967,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.906669020652771,
      "rewards/wrapped_format_reward": 0.125,
      "step": 49
    },
    {
      "completion_length": 500.0,
      "epoch": 10.0,
      "grad_norm": 77.945556640625,
      "kl": 16.699840545654297,
      "learning_rate": 1.5625e-06,
      "loss": 0.668,
      "reward": -0.2709696292877197,
      "reward_std": 4.022421360015869,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.6459696292877197,
      "rewards/wrapped_format_reward": 0.375,
      "step": 50
    },
    {
      "completion_length": 500.0,
      "epoch": 10.2,
      "grad_norm": 11738.6953125,
      "kl": 798.2957763671875,
      "learning_rate": 1.59375e-06,
      "loss": 31.9318,
      "reward": -2.244354724884033,
      "reward_std": 3.1866860389709473,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -2.869354724884033,
      "rewards/wrapped_format_reward": 0.125,
      "step": 51
    },
    {
      "completion_length": 500.0,
      "epoch": 10.4,
      "grad_norm": 14.563969612121582,
      "kl": 5.301497936248779,
      "learning_rate": 1.6250000000000001e-06,
      "loss": 0.2121,
      "reward": 2.282280445098877,
      "reward_std": 0.7978482246398926,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.21771956980228424,
      "rewards/wrapped_format_reward": 0.5,
      "step": 52
    },
    {
      "completion_length": 500.0,
      "epoch": 10.6,
      "grad_norm": 6.280083656311035,
      "kl": 3.3535187244415283,
      "learning_rate": 1.65625e-06,
      "loss": 0.1341,
      "reward": 0.2996126413345337,
      "reward_std": 2.949772357940674,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.3253873586654663,
      "rewards/wrapped_format_reward": 0.125,
      "step": 53
    },
    {
      "completion_length": 500.0,
      "epoch": 10.8,
      "grad_norm": 223484.046875,
      "kl": 25810.041015625,
      "learning_rate": 1.6875000000000001e-06,
      "loss": 1032.4015,
      "reward": -0.344623327255249,
      "reward_std": 3.944869041442871,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.719623327255249,
      "rewards/wrapped_format_reward": 0.375,
      "step": 54
    },
    {
      "completion_length": 500.0,
      "epoch": 11.0,
      "grad_norm": 593881.0,
      "kl": 99004.265625,
      "learning_rate": 1.71875e-06,
      "loss": 3960.1709,
      "reward": -3.875,
      "reward_std": 0.25,
      "rewards/mpc_param_extraction_reward": 0.0,
      "rewards/mpc_param_name_reward": 0.0,
      "rewards/wrapped_driving_reward": -4.0,
      "rewards/wrapped_format_reward": 0.125,
      "step": 55
    },
    {
      "completion_length": 500.0,
      "epoch": 11.2,
      "grad_norm": 6.934082508087158,
      "kl": 2.1647584438323975,
      "learning_rate": 1.75e-06,
      "loss": 0.0866,
      "reward": -1.2983622550964355,
      "reward_std": 2.1767022609710693,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -3.1733622550964355,
      "rewards/wrapped_format_reward": 0.375,
      "step": 56
    },
    {
      "completion_length": 500.0,
      "epoch": 11.4,
      "grad_norm": 11660.44921875,
      "kl": 1039.497802734375,
      "learning_rate": 1.78125e-06,
      "loss": 41.5799,
      "reward": 1.111867070198059,
      "reward_std": 3.4727301597595215,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.6381329298019409,
      "rewards/wrapped_format_reward": 0.25,
      "step": 57
    },
    {
      "completion_length": 500.0,
      "epoch": 11.6,
      "grad_norm": 116.61476135253906,
      "kl": 30.001558303833008,
      "learning_rate": 1.8125e-06,
      "loss": 1.2001,
      "reward": -3.875,
      "reward_std": 0.25,
      "rewards/mpc_param_extraction_reward": 0.0,
      "rewards/mpc_param_name_reward": 0.0,
      "rewards/wrapped_driving_reward": -4.0,
      "rewards/wrapped_format_reward": 0.125,
      "step": 58
    },
    {
      "completion_length": 500.0,
      "epoch": 11.8,
      "grad_norm": 7.750627040863037,
      "kl": 3.2049598693847656,
      "learning_rate": 1.8437500000000003e-06,
      "loss": 0.1282,
      "reward": -3.875,
      "reward_std": 0.25,
      "rewards/mpc_param_extraction_reward": 0.0,
      "rewards/mpc_param_name_reward": 0.0,
      "rewards/wrapped_driving_reward": -4.0,
      "rewards/wrapped_format_reward": 0.125,
      "step": 59
    },
    {
      "completion_length": 500.0,
      "epoch": 12.0,
      "grad_norm": 6.483101844787598,
      "kl": 2.8182482719421387,
      "learning_rate": 1.8750000000000003e-06,
      "loss": 0.1127,
      "reward": 0.4944196343421936,
      "reward_std": 2.8224055767059326,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.3805804252624512,
      "rewards/wrapped_format_reward": 0.375,
      "step": 60
    },
    {
      "completion_length": 500.0,
      "epoch": 12.2,
      "grad_norm": 2.41937255859375,
      "kl": 1.5243698358535767,
      "learning_rate": 1.90625e-06,
      "loss": 0.061,
      "reward": -1.5549238920211792,
      "reward_std": 3.3336286544799805,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.9299237728118896,
      "rewards/wrapped_format_reward": 0.375,
      "step": 61
    },
    {
      "completion_length": 500.0,
      "epoch": 12.4,
      "grad_norm": 1.8835395574569702,
      "kl": 1.3928029537200928,
      "learning_rate": 1.9375e-06,
      "loss": 0.0557,
      "reward": 0.5334538817405701,
      "reward_std": 3.0702548027038574,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.9665461778640747,
      "rewards/wrapped_format_reward": 0.0,
      "step": 62
    },
    {
      "completion_length": 500.0,
      "epoch": 12.6,
      "grad_norm": 82.16962432861328,
      "kl": 9.488784790039062,
      "learning_rate": 1.96875e-06,
      "loss": 0.3796,
      "reward": -0.4010847806930542,
      "reward_std": 4.155675411224365,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.7760847806930542,
      "rewards/wrapped_format_reward": 0.375,
      "step": 63
    },
    {
      "completion_length": 500.0,
      "epoch": 12.8,
      "grad_norm": 1.6330454349517822,
      "kl": 0.7770444750785828,
      "learning_rate": 2.0000000000000003e-06,
      "loss": 0.0311,
      "reward": -1.0365194082260132,
      "reward_std": 3.1399788856506348,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.46875,
      "rewards/wrapped_driving_reward": -2.2552695274353027,
      "rewards/wrapped_format_reward": 0.25,
      "step": 64
    },
    {
      "completion_length": 500.0,
      "epoch": 13.0,
      "grad_norm": 226.30535888671875,
      "kl": 45.10585021972656,
      "learning_rate": 2.0312500000000002e-06,
      "loss": 1.8042,
      "reward": -1.9248578548431396,
      "reward_std": 3.5153682231903076,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -2.7998578548431396,
      "rewards/wrapped_format_reward": 0.375,
      "step": 65
    },
    {
      "completion_length": 500.0,
      "epoch": 13.2,
      "grad_norm": 62.87789535522461,
      "kl": 7.880831718444824,
      "learning_rate": 2.0625e-06,
      "loss": 0.3152,
      "reward": 3.1509861946105957,
      "reward_std": 0.29935261607170105,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.987500011920929,
      "rewards/wrapped_driving_reward": 0.5384860038757324,
      "rewards/wrapped_format_reward": 0.625,
      "step": 66
    },
    {
      "completion_length": 500.0,
      "epoch": 13.4,
      "grad_norm": 41.82035446166992,
      "kl": 9.357061386108398,
      "learning_rate": 2.09375e-06,
      "loss": 0.3743,
      "reward": -2.576190948486328,
      "reward_std": 2.8476178646087646,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -3.201190948486328,
      "rewards/wrapped_format_reward": 0.125,
      "step": 67
    },
    {
      "completion_length": 500.0,
      "epoch": 13.6,
      "grad_norm": 4.275550365447998,
      "kl": 2.5297634601593018,
      "learning_rate": 2.125e-06,
      "loss": 0.1012,
      "reward": -2.036945104598999,
      "reward_std": 3.926109790802002,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -2.786945104598999,
      "rewards/wrapped_format_reward": 0.25,
      "step": 68
    },
    {
      "completion_length": 500.0,
      "epoch": 13.8,
      "grad_norm": 12.396137237548828,
      "kl": 3.0427801609039307,
      "learning_rate": 2.1562500000000003e-06,
      "loss": 0.1217,
      "reward": 0.7283755540847778,
      "reward_std": 3.323927879333496,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.2716244459152222,
      "rewards/wrapped_format_reward": 0.5,
      "step": 69
    },
    {
      "completion_length": 500.0,
      "epoch": 14.0,
      "grad_norm": 1.5282281637191772,
      "kl": 1.092595100402832,
      "learning_rate": 2.1875000000000002e-06,
      "loss": 0.0437,
      "reward": -3.2356114387512207,
      "reward_std": 1.528777003288269,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -3.8606114387512207,
      "rewards/wrapped_format_reward": 0.125,
      "step": 70
    },
    {
      "completion_length": 500.0,
      "epoch": 14.2,
      "grad_norm": 1.2480015754699707,
      "kl": 0.7834239602088928,
      "learning_rate": 2.21875e-06,
      "loss": 0.0313,
      "reward": -2.0355887413024902,
      "reward_std": 3.270659923553467,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -2.7855887413024902,
      "rewards/wrapped_format_reward": 0.25,
      "step": 71
    },
    {
      "completion_length": 500.0,
      "epoch": 14.4,
      "grad_norm": 80.85037231445312,
      "kl": 9.716327667236328,
      "learning_rate": 2.25e-06,
      "loss": 0.3887,
      "reward": -3.375,
      "reward_std": 1.25,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -4.0,
      "rewards/wrapped_format_reward": 0.125,
      "step": 72
    },
    {
      "completion_length": 500.0,
      "epoch": 14.6,
      "grad_norm": 1.1875276565551758,
      "kl": 0.9156450629234314,
      "learning_rate": 2.28125e-06,
      "loss": 0.0366,
      "reward": 0.398318886756897,
      "reward_std": 2.9533674716949463,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.3516812324523926,
      "rewards/wrapped_format_reward": 0.25,
      "step": 73
    },
    {
      "completion_length": 500.0,
      "epoch": 14.8,
      "grad_norm": 5.200348854064941,
      "kl": 2.256690502166748,
      "learning_rate": 2.3125000000000003e-06,
      "loss": 0.0903,
      "reward": -0.17332077026367188,
      "reward_std": 4.147373199462891,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.47727274894714355,
      "rewards/wrapped_driving_reward": -1.650593638420105,
      "rewards/wrapped_format_reward": 0.5,
      "step": 74
    },
    {
      "completion_length": 500.0,
      "epoch": 15.0,
      "grad_norm": 7008.42626953125,
      "kl": 815.8104248046875,
      "learning_rate": 2.3437500000000002e-06,
      "loss": 32.6324,
      "reward": -2.0281713008880615,
      "reward_std": 3.3107235431671143,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -2.9031713008880615,
      "rewards/wrapped_format_reward": 0.375,
      "step": 75
    },
    {
      "completion_length": 500.0,
      "epoch": 15.2,
      "grad_norm": 0.9020519852638245,
      "kl": 0.9176801443099976,
      "learning_rate": 2.375e-06,
      "loss": 0.0367,
      "reward": -2.1365058422088623,
      "reward_std": 3.0964157581329346,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -3.1365058422088623,
      "rewards/wrapped_format_reward": 0.5,
      "step": 76
    },
    {
      "completion_length": 500.0,
      "epoch": 15.4,
      "grad_norm": 2.2519280910491943,
      "kl": 0.8236393332481384,
      "learning_rate": 2.40625e-06,
      "loss": 0.0329,
      "reward": -1.22861909866333,
      "reward_std": 2.9241857528686523,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.353619337081909,
      "rewards/wrapped_format_reward": 0.125,
      "step": 77
    },
    {
      "completion_length": 500.0,
      "epoch": 15.6,
      "grad_norm": 1.5832031965255737,
      "kl": 0.7527546286582947,
      "learning_rate": 2.4375e-06,
      "loss": 0.0301,
      "reward": -0.6693365573883057,
      "reward_std": 3.860503911972046,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.7943366765975952,
      "rewards/wrapped_format_reward": 0.125,
      "step": 78
    },
    {
      "completion_length": 500.0,
      "epoch": 15.8,
      "grad_norm": 1.108726143836975,
      "kl": 1.0248883962631226,
      "learning_rate": 2.4687500000000003e-06,
      "loss": 0.041,
      "reward": -3.125,
      "reward_std": 1.4361406564712524,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -4.0,
      "rewards/wrapped_format_reward": 0.375,
      "step": 79
    },
    {
      "completion_length": 500.0,
      "epoch": 16.0,
      "grad_norm": 1.0169743299484253,
      "kl": 0.7592311501502991,
      "learning_rate": 2.5e-06,
      "loss": 0.0304,
      "reward": 1.145168423652649,
      "reward_std": 3.56965708732605,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.7298316359519958,
      "rewards/wrapped_format_reward": 0.375,
      "step": 80
    },
    {
      "completion_length": 500.0,
      "epoch": 16.2,
      "grad_norm": 0.7822604179382324,
      "kl": 0.560085117816925,
      "learning_rate": 2.53125e-06,
      "loss": 0.0224,
      "reward": 1.2056835889816284,
      "reward_std": 3.5178937911987305,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.6693164110183716,
      "rewards/wrapped_format_reward": 0.375,
      "step": 81
    },
    {
      "completion_length": 500.0,
      "epoch": 16.4,
      "grad_norm": 0.8175077438354492,
      "kl": 0.5752599239349365,
      "learning_rate": 2.5625e-06,
      "loss": 0.023,
      "reward": 2.9252407550811768,
      "reward_std": 0.7892647385597229,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.3002408742904663,
      "rewards/wrapped_format_reward": 0.625,
      "step": 82
    },
    {
      "completion_length": 500.0,
      "epoch": 16.6,
      "grad_norm": 6.47392463684082,
      "kl": 2.1055572032928467,
      "learning_rate": 2.5937500000000004e-06,
      "loss": 0.0842,
      "reward": -2.7708332538604736,
      "reward_std": 1.4678263664245605,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.4791666865348816,
      "rewards/wrapped_driving_reward": -4.0,
      "rewards/wrapped_format_reward": 0.25,
      "step": 83
    },
    {
      "completion_length": 500.0,
      "epoch": 16.8,
      "grad_norm": 0.9220647811889648,
      "kl": 0.7157851457595825,
      "learning_rate": 2.6250000000000003e-06,
      "loss": 0.0286,
      "reward": -2.987729549407959,
      "reward_std": 1.3781793117523193,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -3.987729549407959,
      "rewards/wrapped_format_reward": 0.5,
      "step": 84
    },
    {
      "completion_length": 500.0,
      "epoch": 17.0,
      "grad_norm": 62.959754943847656,
      "kl": 11.148348808288574,
      "learning_rate": 2.65625e-06,
      "loss": 0.4459,
      "reward": -3.875,
      "reward_std": 0.25,
      "rewards/mpc_param_extraction_reward": 0.0,
      "rewards/mpc_param_name_reward": 0.0,
      "rewards/wrapped_driving_reward": -4.0,
      "rewards/wrapped_format_reward": 0.125,
      "step": 85
    },
    {
      "completion_length": 500.0,
      "epoch": 17.2,
      "grad_norm": 1.0555181503295898,
      "kl": 0.7983899712562561,
      "learning_rate": 2.6875e-06,
      "loss": 0.0319,
      "reward": -0.34822893142700195,
      "reward_std": 1.868666648864746,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -2.973228931427002,
      "rewards/wrapped_format_reward": 0.625,
      "step": 86
    },
    {
      "completion_length": 500.0,
      "epoch": 17.4,
      "grad_norm": 5.494264602661133,
      "kl": 1.303008794784546,
      "learning_rate": 2.71875e-06,
      "loss": 0.0521,
      "reward": -1.6160635948181152,
      "reward_std": 3.132638931274414,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.7410635948181152,
      "rewards/wrapped_format_reward": 0.125,
      "step": 87
    },
    {
      "completion_length": 500.0,
      "epoch": 17.6,
      "grad_norm": 4.55032205581665,
      "kl": 0.8609241247177124,
      "learning_rate": 2.7500000000000004e-06,
      "loss": 0.0344,
      "reward": -0.07164722681045532,
      "reward_std": 2.4449591636657715,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.7045454978942871,
      "rewards/wrapped_driving_reward": -1.9011927843093872,
      "rewards/wrapped_format_reward": 0.375,
      "step": 88
    },
    {
      "completion_length": 500.0,
      "epoch": 17.8,
      "grad_norm": 0.9195135831832886,
      "kl": 0.683874785900116,
      "learning_rate": 2.7812500000000003e-06,
      "loss": 0.0274,
      "reward": -0.11890482902526855,
      "reward_std": 3.1548802852630615,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.699999988079071,
      "rewards/wrapped_driving_reward": -2.0689048767089844,
      "rewards/wrapped_format_reward": 0.5,
      "step": 89
    },
    {
      "completion_length": 500.0,
      "epoch": 18.0,
      "grad_norm": 33.38914108276367,
      "kl": 7.705580234527588,
      "learning_rate": 2.8125e-06,
      "loss": 0.3082,
      "reward": 1.586517572402954,
      "reward_std": 3.729795217514038,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.4134823679924011,
      "rewards/wrapped_format_reward": 0.5,
      "step": 90
    },
    {
      "completion_length": 500.0,
      "epoch": 18.2,
      "grad_norm": 57.7512092590332,
      "kl": 6.441009998321533,
      "learning_rate": 2.84375e-06,
      "loss": 0.2576,
      "reward": -0.8143091201782227,
      "reward_std": 3.7249650955200195,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.4444444477558136,
      "rewards/wrapped_driving_reward": -2.008753776550293,
      "rewards/wrapped_format_reward": 0.25,
      "step": 91
    },
    {
      "completion_length": 500.0,
      "epoch": 18.4,
      "grad_norm": 1.0778491497039795,
      "kl": 0.7857025265693665,
      "learning_rate": 2.875e-06,
      "loss": 0.0314,
      "reward": -0.4593994617462158,
      "reward_std": 3.805197238922119,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.8343994617462158,
      "rewards/wrapped_format_reward": 0.375,
      "step": 92
    },
    {
      "completion_length": 500.0,
      "epoch": 18.6,
      "grad_norm": 1.1437242031097412,
      "kl": 0.5162321925163269,
      "learning_rate": 2.9062500000000003e-06,
      "loss": 0.0206,
      "reward": -1.5748236179351807,
      "reward_std": 3.3948116302490234,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.8248236179351807,
      "rewards/wrapped_format_reward": 0.25,
      "step": 93
    },
    {
      "completion_length": 500.0,
      "epoch": 18.8,
      "grad_norm": 0.782410204410553,
      "kl": 0.4215336740016937,
      "learning_rate": 2.9375000000000003e-06,
      "loss": 0.0169,
      "reward": -1.7746977806091309,
      "reward_std": 3.3859715461730957,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.4375,
      "rewards/wrapped_driving_reward": -2.962197780609131,
      "rewards/wrapped_format_reward": 0.25,
      "step": 94
    },
    {
      "completion_length": 500.0,
      "epoch": 19.0,
      "grad_norm": 4.5705718994140625,
      "kl": 2.0152359008789062,
      "learning_rate": 2.96875e-06,
      "loss": 0.0806,
      "reward": 0.6353222131729126,
      "reward_std": 3.1809890270233154,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.1146776676177979,
      "rewards/wrapped_format_reward": 0.25,
      "step": 95
    },
    {
      "completion_length": 500.0,
      "epoch": 19.2,
      "grad_norm": 0.7173673510551453,
      "kl": 0.5042878985404968,
      "learning_rate": 3e-06,
      "loss": 0.0202,
      "reward": 3.3471288681030273,
      "reward_std": 0.31114432215690613,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.722128689289093,
      "rewards/wrapped_format_reward": 0.625,
      "step": 96
    },
    {
      "completion_length": 500.0,
      "epoch": 19.4,
      "grad_norm": 2.924496650695801,
      "kl": 0.9709882736206055,
      "learning_rate": 3.03125e-06,
      "loss": 0.0388,
      "reward": 1.2860008478164673,
      "reward_std": 2.1523053646087646,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9166666865348816,
      "rewards/wrapped_driving_reward": -1.1306657791137695,
      "rewards/wrapped_format_reward": 0.5,
      "step": 97
    },
    {
      "completion_length": 500.0,
      "epoch": 19.6,
      "grad_norm": 1.3449220657348633,
      "kl": 0.776192843914032,
      "learning_rate": 3.0625000000000003e-06,
      "loss": 0.031,
      "reward": -1.2475244998931885,
      "reward_std": 3.184887170791626,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.2475244998931885,
      "rewards/wrapped_format_reward": 0.0,
      "step": 98
    },
    {
      "completion_length": 500.0,
      "epoch": 19.8,
      "grad_norm": 16.095233917236328,
      "kl": 1.6037352085113525,
      "learning_rate": 3.0937500000000002e-06,
      "loss": 0.0641,
      "reward": 0.7092133164405823,
      "reward_std": 3.2424275875091553,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.040786862373352,
      "rewards/wrapped_format_reward": 0.25,
      "step": 99
    },
    {
      "completion_length": 500.0,
      "epoch": 20.0,
      "grad_norm": 1.065063714981079,
      "kl": 0.6967657208442688,
      "learning_rate": 3.125e-06,
      "loss": 0.0279,
      "reward": 1.58005690574646,
      "reward_std": 3.7297377586364746,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.41994309425354004,
      "rewards/wrapped_format_reward": 0.5,
      "step": 100
    },
    {
      "completion_length": 500.0,
      "epoch": 20.2,
      "grad_norm": 0.7442240715026855,
      "kl": 0.5057598352432251,
      "learning_rate": 3.15625e-06,
      "loss": 0.0202,
      "reward": -0.4889770746231079,
      "reward_std": 3.2432987689971924,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.1139769554138184,
      "rewards/wrapped_format_reward": 0.625,
      "step": 101
    },
    {
      "completion_length": 500.0,
      "epoch": 20.4,
      "grad_norm": 0.6366997361183167,
      "kl": 0.44367504119873047,
      "learning_rate": 3.1875e-06,
      "loss": 0.0177,
      "reward": -2.3750693798065186,
      "reward_std": 2.5939254760742188,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -3.2500693798065186,
      "rewards/wrapped_format_reward": 0.375,
      "step": 102
    },
    {
      "completion_length": 500.0,
      "epoch": 20.6,
      "grad_norm": 2.0096611976623535,
      "kl": 0.4689376652240753,
      "learning_rate": 3.2187500000000003e-06,
      "loss": 0.0188,
      "reward": -0.5932518243789673,
      "reward_std": 3.942629814147949,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.9682518243789673,
      "rewards/wrapped_format_reward": 0.375,
      "step": 103
    },
    {
      "completion_length": 500.0,
      "epoch": 20.8,
      "grad_norm": 1.2003757953643799,
      "kl": 0.3688035309314728,
      "learning_rate": 3.2500000000000002e-06,
      "loss": 0.0148,
      "reward": 2.4348607063293457,
      "reward_std": 1.402535080909729,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.824999988079071,
      "rewards/wrapped_driving_reward": 0.1098608672618866,
      "rewards/wrapped_format_reward": 0.5,
      "step": 104
    },
    {
      "completion_length": 500.0,
      "epoch": 21.0,
      "grad_norm": 1.0951755046844482,
      "kl": 0.6130416393280029,
      "learning_rate": 3.28125e-06,
      "loss": 0.0245,
      "reward": 1.912153720855713,
      "reward_std": 2.661609649658203,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.4628463387489319,
      "rewards/wrapped_format_reward": 0.375,
      "step": 105
    },
    {
      "completion_length": 500.0,
      "epoch": 21.2,
      "grad_norm": 0.7108362317085266,
      "kl": 0.4603574275970459,
      "learning_rate": 3.3125e-06,
      "loss": 0.0184,
      "reward": -0.09991639852523804,
      "reward_std": 2.701847791671753,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.699999988079071,
      "rewards/wrapped_driving_reward": -1.674916386604309,
      "rewards/wrapped_format_reward": 0.125,
      "step": 106
    },
    {
      "completion_length": 500.0,
      "epoch": 21.4,
      "grad_norm": 0.6701599955558777,
      "kl": 0.40188899636268616,
      "learning_rate": 3.34375e-06,
      "loss": 0.0161,
      "reward": -0.8211934566497803,
      "reward_std": 3.6706011295318604,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.0711934566497803,
      "rewards/wrapped_format_reward": 0.25,
      "step": 107
    },
    {
      "completion_length": 500.0,
      "epoch": 21.6,
      "grad_norm": 2.1547489166259766,
      "kl": 1.2839192152023315,
      "learning_rate": 3.3750000000000003e-06,
      "loss": 0.0514,
      "reward": 1.2595562934875488,
      "reward_std": 3.514036178588867,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.36544373631477356,
      "rewards/wrapped_format_reward": 0.125,
      "step": 108
    },
    {
      "completion_length": 500.0,
      "epoch": 21.8,
      "grad_norm": 1.838152527809143,
      "kl": 0.49252963066101074,
      "learning_rate": 3.40625e-06,
      "loss": 0.0197,
      "reward": 1.507678508758545,
      "reward_std": 3.675663948059082,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.7272727489471436,
      "rewards/wrapped_driving_reward": -0.46959418058395386,
      "rewards/wrapped_format_reward": 0.5,
      "step": 109
    },
    {
      "completion_length": 500.0,
      "epoch": 22.0,
      "grad_norm": 0.7439582347869873,
      "kl": 0.595367431640625,
      "learning_rate": 3.4375e-06,
      "loss": 0.0238,
      "reward": -3.625,
      "reward_std": 0.4787135720252991,
      "rewards/mpc_param_extraction_reward": 0.0,
      "rewards/mpc_param_name_reward": 0.0,
      "rewards/wrapped_driving_reward": -4.0,
      "rewards/wrapped_format_reward": 0.375,
      "step": 110
    },
    {
      "completion_length": 500.0,
      "epoch": 22.2,
      "grad_norm": 0.5864555835723877,
      "kl": 0.3803044855594635,
      "learning_rate": 3.46875e-06,
      "loss": 0.0152,
      "reward": 1.1622142791748047,
      "reward_std": 3.198068618774414,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.7272727489471436,
      "rewards/wrapped_driving_reward": -1.0650583505630493,
      "rewards/wrapped_format_reward": 0.75,
      "step": 111
    },
    {
      "completion_length": 500.0,
      "epoch": 22.4,
      "grad_norm": 4.526273250579834,
      "kl": 0.46392467617988586,
      "learning_rate": 3.5e-06,
      "loss": 0.0186,
      "reward": -0.7247750163078308,
      "reward_std": 3.514815330505371,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.0997748374938965,
      "rewards/wrapped_format_reward": 0.375,
      "step": 112
    },
    {
      "completion_length": 500.0,
      "epoch": 22.6,
      "grad_norm": 0.5704318881034851,
      "kl": 0.31563645601272583,
      "learning_rate": 3.5312500000000007e-06,
      "loss": 0.0126,
      "reward": -0.6594128608703613,
      "reward_std": 3.574946880340576,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.0344128608703613,
      "rewards/wrapped_format_reward": 0.375,
      "step": 113
    },
    {
      "completion_length": 500.0,
      "epoch": 22.8,
      "grad_norm": 0.671708881855011,
      "kl": 0.44867807626724243,
      "learning_rate": 3.5625e-06,
      "loss": 0.0179,
      "reward": 1.6787878274917603,
      "reward_std": 0.9073445200920105,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.9462121725082397,
      "rewards/wrapped_format_reward": 0.625,
      "step": 114
    },
    {
      "completion_length": 500.0,
      "epoch": 23.0,
      "grad_norm": 1.2352200746536255,
      "kl": 0.46100977063179016,
      "learning_rate": 3.59375e-06,
      "loss": 0.0184,
      "reward": -0.30863749980926514,
      "reward_std": 3.9797427654266357,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.8086374998092651,
      "rewards/wrapped_format_reward": 0.5,
      "step": 115
    },
    {
      "completion_length": 500.0,
      "epoch": 23.2,
      "grad_norm": 0.783157229423523,
      "kl": 0.44468817114830017,
      "learning_rate": 3.625e-06,
      "loss": 0.0178,
      "reward": 1.9851404428482056,
      "reward_std": 1.0498998165130615,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.2648596167564392,
      "rewards/wrapped_format_reward": 0.25,
      "step": 116
    },
    {
      "completion_length": 500.0,
      "epoch": 23.4,
      "grad_norm": 0.6096097826957703,
      "kl": 0.36140069365501404,
      "learning_rate": 3.65625e-06,
      "loss": 0.0145,
      "reward": -0.9730753898620605,
      "reward_std": 2.724126100540161,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -3.0980753898620605,
      "rewards/wrapped_format_reward": 0.625,
      "step": 117
    },
    {
      "completion_length": 500.0,
      "epoch": 23.6,
      "grad_norm": 2.4817147254943848,
      "kl": 0.3356289267539978,
      "learning_rate": 3.6875000000000007e-06,
      "loss": 0.0134,
      "reward": 2.9043874740600586,
      "reward_std": 0.34505343437194824,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.5293872952461243,
      "rewards/wrapped_format_reward": 0.375,
      "step": 118
    },
    {
      "completion_length": 500.0,
      "epoch": 23.8,
      "grad_norm": 1.3457905054092407,
      "kl": 0.32610735297203064,
      "learning_rate": 3.7187500000000006e-06,
      "loss": 0.013,
      "reward": -0.004844188690185547,
      "reward_std": 3.559382915496826,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.7548441886901855,
      "rewards/wrapped_format_reward": 0.25,
      "step": 119
    },
    {
      "completion_length": 500.0,
      "epoch": 24.0,
      "grad_norm": 0.8271002769470215,
      "kl": 0.592341423034668,
      "learning_rate": 3.7500000000000005e-06,
      "loss": 0.0237,
      "reward": 0.23737984895706177,
      "reward_std": 2.8921873569488525,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.6000000238418579,
      "rewards/wrapped_driving_reward": -1.2376201152801514,
      "rewards/wrapped_format_reward": 0.125,
      "step": 120
    },
    {
      "completion_length": 500.0,
      "epoch": 24.2,
      "grad_norm": 1.575377106666565,
      "kl": 0.31468361616134644,
      "learning_rate": 3.78125e-06,
      "loss": 0.0126,
      "reward": 0.08798408508300781,
      "reward_std": 3.364243984222412,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.9120157957077026,
      "rewards/wrapped_format_reward": 0.5,
      "step": 121
    },
    {
      "completion_length": 500.0,
      "epoch": 24.4,
      "grad_norm": 1.548542857170105,
      "kl": 0.7125066518783569,
      "learning_rate": 3.8125e-06,
      "loss": 0.0285,
      "reward": 3.202035903930664,
      "reward_std": 0.5515704154968262,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.8958333134651184,
      "rewards/wrapped_driving_reward": 0.6812027096748352,
      "rewards/wrapped_format_reward": 0.625,
      "step": 122
    },
    {
      "completion_length": 500.0,
      "epoch": 24.6,
      "grad_norm": 0.6466585397720337,
      "kl": 0.33141595125198364,
      "learning_rate": 3.84375e-06,
      "loss": 0.0133,
      "reward": -0.8563422560691833,
      "reward_std": 2.9308738708496094,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -2.981342315673828,
      "rewards/wrapped_format_reward": 0.625,
      "step": 123
    },
    {
      "completion_length": 500.0,
      "epoch": 24.8,
      "grad_norm": 0.9053751826286316,
      "kl": 0.3941192626953125,
      "learning_rate": 3.875e-06,
      "loss": 0.0158,
      "reward": -0.9088470935821533,
      "reward_std": 2.4116313457489014,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -2.6588470935821533,
      "rewards/wrapped_format_reward": 0.25,
      "step": 124
    },
    {
      "completion_length": 500.0,
      "epoch": 25.0,
      "grad_norm": 0.7404253482818604,
      "kl": 0.3537856936454773,
      "learning_rate": 3.90625e-06,
      "loss": 0.0142,
      "reward": -0.08935052156448364,
      "reward_std": 4.237273693084717,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.5893504619598389,
      "rewards/wrapped_format_reward": 0.5,
      "step": 125
    },
    {
      "completion_length": 500.0,
      "epoch": 25.2,
      "grad_norm": 0.5974608659744263,
      "kl": 0.31292691826820374,
      "learning_rate": 3.9375e-06,
      "loss": 0.0125,
      "reward": -0.5100458860397339,
      "reward_std": 3.746746063232422,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.8850458860397339,
      "rewards/wrapped_format_reward": 0.375,
      "step": 126
    },
    {
      "completion_length": 500.0,
      "epoch": 25.4,
      "grad_norm": 0.9886866807937622,
      "kl": 0.3266676068305969,
      "learning_rate": 3.96875e-06,
      "loss": 0.0131,
      "reward": 3.5397558212280273,
      "reward_std": 0.24529722332954407,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.7897558212280273,
      "rewards/wrapped_format_reward": 0.75,
      "step": 127
    },
    {
      "completion_length": 500.0,
      "epoch": 25.6,
      "grad_norm": 0.6569087505340576,
      "kl": 0.28314509987831116,
      "learning_rate": 4.000000000000001e-06,
      "loss": 0.0113,
      "reward": -0.5303106904029846,
      "reward_std": 4.0666303634643555,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.375,
      "rewards/wrapped_driving_reward": -1.7803106307983398,
      "rewards/wrapped_format_reward": 0.375,
      "step": 128
    },
    {
      "completion_length": 500.0,
      "epoch": 25.8,
      "grad_norm": 1.4679771661758423,
      "kl": 0.4160246253013611,
      "learning_rate": 4.031250000000001e-06,
      "loss": 0.0166,
      "reward": -0.5868573188781738,
      "reward_std": 3.941485643386841,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.8368571996688843,
      "rewards/wrapped_format_reward": 0.25,
      "step": 129
    },
    {
      "completion_length": 500.0,
      "epoch": 26.0,
      "grad_norm": 0.5985941290855408,
      "kl": 0.31736528873443604,
      "learning_rate": 4.0625000000000005e-06,
      "loss": 0.0127,
      "reward": 1.1113789081573486,
      "reward_std": 3.429651975631714,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.0136209726333618,
      "rewards/wrapped_format_reward": 0.625,
      "step": 130
    },
    {
      "completion_length": 500.0,
      "epoch": 26.2,
      "grad_norm": 0.6327362656593323,
      "kl": 0.40226221084594727,
      "learning_rate": 4.09375e-06,
      "loss": 0.0161,
      "reward": 0.7944153547286987,
      "reward_std": 2.8826069831848145,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.7250000238418579,
      "rewards/wrapped_driving_reward": -1.3055846691131592,
      "rewards/wrapped_format_reward": 0.625,
      "step": 131
    },
    {
      "completion_length": 500.0,
      "epoch": 26.4,
      "grad_norm": 0.6797881722450256,
      "kl": 0.4582635164260864,
      "learning_rate": 4.125e-06,
      "loss": 0.0183,
      "reward": 2.8031327724456787,
      "reward_std": 0.7006269097328186,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.05313277989625931,
      "rewards/wrapped_format_reward": 0.75,
      "step": 132
    },
    {
      "completion_length": 500.0,
      "epoch": 26.6,
      "grad_norm": 0.5752917528152466,
      "kl": 0.36153456568717957,
      "learning_rate": 4.15625e-06,
      "loss": 0.0145,
      "reward": -2.564105987548828,
      "reward_std": 2.871788263320923,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -3.189105987548828,
      "rewards/wrapped_format_reward": 0.125,
      "step": 133
    },
    {
      "completion_length": 500.0,
      "epoch": 26.8,
      "grad_norm": 0.569823145866394,
      "kl": 0.3600581884384155,
      "learning_rate": 4.1875e-06,
      "loss": 0.0144,
      "reward": 3.2037861347198486,
      "reward_std": 0.1732039451599121,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.8287861943244934,
      "rewards/wrapped_format_reward": 0.375,
      "step": 134
    },
    {
      "completion_length": 500.0,
      "epoch": 27.0,
      "grad_norm": 11.942618370056152,
      "kl": 2.177290678024292,
      "learning_rate": 4.21875e-06,
      "loss": 0.0871,
      "reward": -2.3714582920074463,
      "reward_std": 1.8921570777893066,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -3.7464582920074463,
      "rewards/wrapped_format_reward": 0.375,
      "step": 135
    },
    {
      "completion_length": 500.0,
      "epoch": 27.2,
      "grad_norm": 0.5660642385482788,
      "kl": 0.2908819019794464,
      "learning_rate": 4.25e-06,
      "loss": 0.0116,
      "reward": -0.6192033290863037,
      "reward_std": 3.6331570148468018,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.1192033290863037,
      "rewards/wrapped_format_reward": 0.5,
      "step": 136
    },
    {
      "completion_length": 500.0,
      "epoch": 27.4,
      "grad_norm": 1.4041975736618042,
      "kl": 0.463067889213562,
      "learning_rate": 4.28125e-06,
      "loss": 0.0185,
      "reward": -2.75,
      "reward_std": 1.1902379989624023,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -4.0,
      "rewards/wrapped_format_reward": 0.25,
      "step": 137
    },
    {
      "completion_length": 500.0,
      "epoch": 27.6,
      "grad_norm": 0.4801470637321472,
      "kl": 0.2532914876937866,
      "learning_rate": 4.312500000000001e-06,
      "loss": 0.0101,
      "reward": -2.304798126220703,
      "reward_std": 3.3904037475585938,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -3.054798126220703,
      "rewards/wrapped_format_reward": 0.25,
      "step": 138
    },
    {
      "completion_length": 500.0,
      "epoch": 27.8,
      "grad_norm": 0.6999854445457458,
      "kl": 0.4938638210296631,
      "learning_rate": 4.3437500000000006e-06,
      "loss": 0.0198,
      "reward": -1.9556548595428467,
      "reward_std": 3.430131196975708,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -2.9556548595428467,
      "rewards/wrapped_format_reward": 0.5,
      "step": 139
    },
    {
      "completion_length": 500.0,
      "epoch": 28.0,
      "grad_norm": 1.7622352838516235,
      "kl": 0.32535520195961,
      "learning_rate": 4.3750000000000005e-06,
      "loss": 0.013,
      "reward": 3.048956871032715,
      "reward_std": 0.7497459053993225,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.42395687103271484,
      "rewards/wrapped_format_reward": 0.625,
      "step": 140
    },
    {
      "completion_length": 500.0,
      "epoch": 28.2,
      "grad_norm": 1.0910435914993286,
      "kl": 0.3166691064834595,
      "learning_rate": 4.40625e-06,
      "loss": 0.0127,
      "reward": 2.1717541217803955,
      "reward_std": 2.45133900642395,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.45324593782424927,
      "rewards/wrapped_format_reward": 0.625,
      "step": 141
    },
    {
      "completion_length": 500.0,
      "epoch": 28.4,
      "grad_norm": 0.563035249710083,
      "kl": 0.34334975481033325,
      "learning_rate": 4.4375e-06,
      "loss": 0.0137,
      "reward": -0.30545544624328613,
      "reward_std": 2.531362295150757,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.71875,
      "rewards/wrapped_driving_reward": -2.149205446243286,
      "rewards/wrapped_format_reward": 0.375,
      "step": 142
    },
    {
      "completion_length": 500.0,
      "epoch": 28.6,
      "grad_norm": 0.6513370871543884,
      "kl": 0.2893451154232025,
      "learning_rate": 4.46875e-06,
      "loss": 0.0116,
      "reward": -0.7655331492424011,
      "reward_std": 3.818908214569092,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.265533208847046,
      "rewards/wrapped_format_reward": 0.5,
      "step": 143
    },
    {
      "completion_length": 500.0,
      "epoch": 28.8,
      "grad_norm": 0.6747258305549622,
      "kl": 0.4012701213359833,
      "learning_rate": 4.5e-06,
      "loss": 0.0161,
      "reward": -1.631712794303894,
      "reward_std": 2.7382092475891113,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -3.2567129135131836,
      "rewards/wrapped_format_reward": 0.625,
      "step": 144
    },
    {
      "completion_length": 500.0,
      "epoch": 29.0,
      "grad_norm": 1.388415813446045,
      "kl": 0.3030587136745453,
      "learning_rate": 4.53125e-06,
      "loss": 0.0121,
      "reward": -0.3709021210670471,
      "reward_std": 3.0999691486358643,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.6666666865348816,
      "rewards/wrapped_driving_reward": -2.0375688076019287,
      "rewards/wrapped_format_reward": 0.25,
      "step": 145
    },
    {
      "completion_length": 500.0,
      "epoch": 29.2,
      "grad_norm": 1.3835958242416382,
      "kl": 0.5185285806655884,
      "learning_rate": 4.5625e-06,
      "loss": 0.0207,
      "reward": -0.25881457328796387,
      "reward_std": 2.5411531925201416,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9010416865348816,
      "rewards/wrapped_driving_reward": -2.6598563194274902,
      "rewards/wrapped_format_reward": 0.5,
      "step": 146
    },
    {
      "completion_length": 500.0,
      "epoch": 29.4,
      "grad_norm": 1.0529229640960693,
      "kl": 0.304034560918808,
      "learning_rate": 4.59375e-06,
      "loss": 0.0122,
      "reward": 0.5033270120620728,
      "reward_std": 3.8062596321105957,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.6216729879379272,
      "rewards/wrapped_format_reward": 0.625,
      "step": 147
    },
    {
      "completion_length": 500.0,
      "epoch": 29.6,
      "grad_norm": 1.3924496173858643,
      "kl": 0.3519279956817627,
      "learning_rate": 4.625000000000001e-06,
      "loss": 0.0141,
      "reward": 1.081155776977539,
      "reward_std": 2.07023286819458,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9750000238418579,
      "rewards/wrapped_driving_reward": -1.1438441276550293,
      "rewards/wrapped_format_reward": 0.25,
      "step": 148
    },
    {
      "completion_length": 500.0,
      "epoch": 29.8,
      "grad_norm": 1.6642379760742188,
      "kl": 0.5595217347145081,
      "learning_rate": 4.6562500000000005e-06,
      "loss": 0.0224,
      "reward": 2.879631519317627,
      "reward_std": 0.5703426003456116,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.12963154911994934,
      "rewards/wrapped_format_reward": 0.75,
      "step": 149
    },
    {
      "completion_length": 500.0,
      "epoch": 30.0,
      "grad_norm": 0.5775982737541199,
      "kl": 0.2810514271259308,
      "learning_rate": 4.6875000000000004e-06,
      "loss": 0.0112,
      "reward": 0.10444420576095581,
      "reward_std": 3.2514774799346924,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -2.0205557346343994,
      "rewards/wrapped_format_reward": 0.625,
      "step": 150
    },
    {
      "completion_length": 500.0,
      "epoch": 30.2,
      "grad_norm": 0.9198185801506042,
      "kl": 0.28956174850463867,
      "learning_rate": 4.71875e-06,
      "loss": 0.0116,
      "reward": -0.260436087846756,
      "reward_std": 2.8927173614501953,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -2.2604360580444336,
      "rewards/wrapped_format_reward": 0.5,
      "step": 151
    },
    {
      "completion_length": 500.0,
      "epoch": 30.4,
      "grad_norm": 0.7754166722297668,
      "kl": 0.38463443517684937,
      "learning_rate": 4.75e-06,
      "loss": 0.0154,
      "reward": 0.4493406414985657,
      "reward_std": 2.646808385848999,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.300659418106079,
      "rewards/wrapped_format_reward": 0.25,
      "step": 152
    },
    {
      "completion_length": 500.0,
      "epoch": 30.6,
      "grad_norm": 0.5780096650123596,
      "kl": 0.3385607898235321,
      "learning_rate": 4.781250000000001e-06,
      "loss": 0.0135,
      "reward": -1.7927734851837158,
      "reward_std": 3.755190849304199,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -2.792773485183716,
      "rewards/wrapped_format_reward": 0.5,
      "step": 153
    },
    {
      "completion_length": 500.0,
      "epoch": 30.8,
      "grad_norm": 0.5552729964256287,
      "kl": 0.28436288237571716,
      "learning_rate": 4.8125e-06,
      "loss": 0.0114,
      "reward": 0.6222386360168457,
      "reward_std": 2.1850173473358154,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -1.8777613639831543,
      "rewards/wrapped_format_reward": 0.5,
      "step": 154
    },
    {
      "completion_length": 500.0,
      "epoch": 31.0,
      "grad_norm": 0.9199939370155334,
      "kl": 0.37593454122543335,
      "learning_rate": 4.84375e-06,
      "loss": 0.015,
      "reward": 0.4306233525276184,
      "reward_std": 3.5992963314056396,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.6607142686843872,
      "rewards/wrapped_driving_reward": -1.605090856552124,
      "rewards/wrapped_format_reward": 0.625,
      "step": 155
    },
    {
      "completion_length": 500.0,
      "epoch": 31.2,
      "grad_norm": 0.5603945851325989,
      "kl": 0.3141997754573822,
      "learning_rate": 4.875e-06,
      "loss": 0.0126,
      "reward": -2.0,
      "reward_std": 1.3540064096450806,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -4.0,
      "rewards/wrapped_format_reward": 0.5,
      "step": 156
    },
    {
      "completion_length": 500.0,
      "epoch": 31.4,
      "grad_norm": 0.6190344095230103,
      "kl": 0.27537742257118225,
      "learning_rate": 4.90625e-06,
      "loss": 0.011,
      "reward": 1.0282058715820312,
      "reward_std": 3.394833564758301,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.846794068813324,
      "rewards/wrapped_format_reward": 0.375,
      "step": 157
    },
    {
      "completion_length": 500.0,
      "epoch": 31.6,
      "grad_norm": 0.6877399682998657,
      "kl": 0.2958383858203888,
      "learning_rate": 4.937500000000001e-06,
      "loss": 0.0118,
      "reward": -0.755041241645813,
      "reward_std": 3.4646472930908203,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.1300413608551025,
      "rewards/wrapped_format_reward": 0.375,
      "step": 158
    },
    {
      "completion_length": 500.0,
      "epoch": 31.8,
      "grad_norm": 31.595932006835938,
      "kl": 6.78364372253418,
      "learning_rate": 4.9687500000000005e-06,
      "loss": 0.2713,
      "reward": 0.35857605934143066,
      "reward_std": 2.9158554077148438,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.7664239406585693,
      "rewards/wrapped_format_reward": 0.625,
      "step": 159
    },
    {
      "completion_length": 500.0,
      "epoch": 32.0,
      "grad_norm": 1.7048529386520386,
      "kl": 0.35252463817596436,
      "learning_rate": 5e-06,
      "loss": 0.0141,
      "reward": 2.64233136177063,
      "reward_std": 0.7985239624977112,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.14233140647411346,
      "rewards/wrapped_format_reward": 0.5,
      "step": 160
    },
    {
      "completion_length": 500.0,
      "epoch": 32.2,
      "grad_norm": 0.9699507355690002,
      "kl": 0.3963090479373932,
      "learning_rate": 4.99999405044338e-06,
      "loss": 0.0159,
      "reward": 2.578547477722168,
      "reward_std": 0.16937123239040375,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.046452634036540985,
      "rewards/wrapped_format_reward": 0.625,
      "step": 161
    },
    {
      "completion_length": 500.0,
      "epoch": 32.4,
      "grad_norm": 0.6427643299102783,
      "kl": 0.2770542800426483,
      "learning_rate": 4.999976201801837e-06,
      "loss": 0.0111,
      "reward": 2.2058539390563965,
      "reward_std": 1.1022424697875977,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.41914597153663635,
      "rewards/wrapped_format_reward": 0.625,
      "step": 162
    },
    {
      "completion_length": 500.0,
      "epoch": 32.6,
      "grad_norm": 0.6896190047264099,
      "kl": 0.27440541982650757,
      "learning_rate": 4.999946454160323e-06,
      "loss": 0.011,
      "reward": 1.1675429344177246,
      "reward_std": 3.4587650299072266,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.7142857313156128,
      "rewards/wrapped_driving_reward": -0.7967426180839539,
      "rewards/wrapped_format_reward": 0.5,
      "step": 163
    },
    {
      "completion_length": 500.0,
      "epoch": 32.8,
      "grad_norm": 0.7695831060409546,
      "kl": 0.4198772609233856,
      "learning_rate": 4.9999048076604286e-06,
      "loss": 0.0168,
      "reward": -1.912316918373108,
      "reward_std": 2.555265426635742,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.9123167991638184,
      "rewards/wrapped_format_reward": 0.0,
      "step": 164
    },
    {
      "completion_length": 500.0,
      "epoch": 33.0,
      "grad_norm": 0.5920954942703247,
      "kl": 0.28969520330429077,
      "learning_rate": 4.999851262500375e-06,
      "loss": 0.0116,
      "reward": 3.1377110481262207,
      "reward_std": 0.5497580170631409,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.5127109289169312,
      "rewards/wrapped_format_reward": 0.625,
      "step": 165
    },
    {
      "completion_length": 500.0,
      "epoch": 33.2,
      "grad_norm": 6.077564716339111,
      "kl": 0.44747114181518555,
      "learning_rate": 4.999785818935018e-06,
      "loss": 0.0179,
      "reward": 2.047877073287964,
      "reward_std": 2.722182035446167,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.45212286710739136,
      "rewards/wrapped_format_reward": 0.5,
      "step": 166
    },
    {
      "completion_length": 500.0,
      "epoch": 33.4,
      "grad_norm": 0.6889002919197083,
      "kl": 0.37658053636550903,
      "learning_rate": 4.999708477275846e-06,
      "loss": 0.0151,
      "reward": -2.284590482711792,
      "reward_std": 3.106440782546997,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -3.159590482711792,
      "rewards/wrapped_format_reward": 0.375,
      "step": 167
    },
    {
      "completion_length": 500.0,
      "epoch": 33.6,
      "grad_norm": 1.8645473718643188,
      "kl": 0.3408987522125244,
      "learning_rate": 4.9996192378909785e-06,
      "loss": 0.0136,
      "reward": 0.917718231678009,
      "reward_std": 2.948974132537842,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.699999988079071,
      "rewards/wrapped_driving_reward": -1.157281756401062,
      "rewards/wrapped_format_reward": 0.625,
      "step": 168
    },
    {
      "completion_length": 500.0,
      "epoch": 33.8,
      "grad_norm": 0.535763680934906,
      "kl": 0.25453072786331177,
      "learning_rate": 4.999518101205162e-06,
      "loss": 0.0102,
      "reward": 3.604552745819092,
      "reward_std": 0.45598289370536804,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.7295528054237366,
      "rewards/wrapped_format_reward": 0.875,
      "step": 169
    },
    {
      "completion_length": 500.0,
      "epoch": 34.0,
      "grad_norm": 1.0853776931762695,
      "kl": 0.2871979773044586,
      "learning_rate": 4.999405067699773e-06,
      "loss": 0.0115,
      "reward": 0.7697337865829468,
      "reward_std": 3.0176069736480713,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.3552662134170532,
      "rewards/wrapped_format_reward": 0.625,
      "step": 170
    },
    {
      "completion_length": 500.0,
      "epoch": 34.2,
      "grad_norm": 2.175551176071167,
      "kl": 0.7303879261016846,
      "learning_rate": 4.99928013791281e-06,
      "loss": 0.0292,
      "reward": 0.010015249252319336,
      "reward_std": 4.346557140350342,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.46875,
      "rewards/wrapped_driving_reward": -1.5837347507476807,
      "rewards/wrapped_format_reward": 0.625,
      "step": 171
    },
    {
      "completion_length": 500.0,
      "epoch": 34.4,
      "grad_norm": 1.3378883600234985,
      "kl": 0.2555471658706665,
      "learning_rate": 4.999143312438893e-06,
      "loss": 0.0102,
      "reward": 1.064118504524231,
      "reward_std": 1.464298963546753,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -1.4358813762664795,
      "rewards/wrapped_format_reward": 0.5,
      "step": 172
    },
    {
      "completion_length": 500.0,
      "epoch": 34.6,
      "grad_norm": 1.6005758047103882,
      "kl": 0.3272940516471863,
      "learning_rate": 4.998994591929266e-06,
      "loss": 0.0131,
      "reward": 3.320277214050293,
      "reward_std": 0.5942137241363525,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.8202772736549377,
      "rewards/wrapped_format_reward": 0.5,
      "step": 173
    },
    {
      "completion_length": 500.0,
      "epoch": 34.8,
      "grad_norm": 0.8775622844696045,
      "kl": 0.3981474041938782,
      "learning_rate": 4.998833977091783e-06,
      "loss": 0.0159,
      "reward": 2.548191547393799,
      "reward_std": 0.13038182258605957,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.17319151759147644,
      "rewards/wrapped_format_reward": 0.375,
      "step": 174
    },
    {
      "completion_length": 500.0,
      "epoch": 35.0,
      "grad_norm": 0.5131356716156006,
      "kl": 0.26495081186294556,
      "learning_rate": 4.998661468690914e-06,
      "loss": 0.0106,
      "reward": 0.2881455421447754,
      "reward_std": 3.1594552993774414,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.7118544578552246,
      "rewards/wrapped_format_reward": 0.5,
      "step": 175
    },
    {
      "completion_length": 500.0,
      "epoch": 35.2,
      "grad_norm": 1.4990577697753906,
      "kl": 0.3656232953071594,
      "learning_rate": 4.99847706754774e-06,
      "loss": 0.0146,
      "reward": 2.0933961868286133,
      "reward_std": 0.39702948927879333,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9375,
      "rewards/wrapped_driving_reward": -0.594103991985321,
      "rewards/wrapped_format_reward": 0.75,
      "step": 176
    },
    {
      "completion_length": 500.0,
      "epoch": 35.4,
      "grad_norm": 0.5740483999252319,
      "kl": 0.265653520822525,
      "learning_rate": 4.998280774539943e-06,
      "loss": 0.0106,
      "reward": 1.1700050830841064,
      "reward_std": 3.1657402515411377,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.8299949765205383,
      "rewards/wrapped_format_reward": 0.5,
      "step": 177
    },
    {
      "completion_length": 500.0,
      "epoch": 35.6,
      "grad_norm": 0.6564896702766418,
      "kl": 0.265337198972702,
      "learning_rate": 4.998072590601808e-06,
      "loss": 0.0106,
      "reward": -0.852949857711792,
      "reward_std": 3.3822429180145264,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.102949857711792,
      "rewards/wrapped_format_reward": 0.25,
      "step": 178
    },
    {
      "completion_length": 500.0,
      "epoch": 35.8,
      "grad_norm": 23.83641242980957,
      "kl": 4.303451061248779,
      "learning_rate": 4.9978525167242176e-06,
      "loss": 0.1721,
      "reward": 0.764412522315979,
      "reward_std": 2.8684115409851074,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.7250000238418579,
      "rewards/wrapped_driving_reward": -1.085587501525879,
      "rewards/wrapped_format_reward": 0.375,
      "step": 179
    },
    {
      "completion_length": 500.0,
      "epoch": 36.0,
      "grad_norm": 0.7350974082946777,
      "kl": 0.30466321110725403,
      "learning_rate": 4.997620553954645e-06,
      "loss": 0.0122,
      "reward": -0.10997164249420166,
      "reward_std": 2.883012056350708,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -2.109971523284912,
      "rewards/wrapped_format_reward": 0.5,
      "step": 180
    },
    {
      "completion_length": 500.0,
      "epoch": 36.2,
      "grad_norm": 1.8978265523910522,
      "kl": 0.5050737857818604,
      "learning_rate": 4.997376703397151e-06,
      "loss": 0.0202,
      "reward": -0.35431569814682007,
      "reward_std": 4.209678649902344,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.8543156385421753,
      "rewards/wrapped_format_reward": 0.5,
      "step": 181
    },
    {
      "completion_length": 500.0,
      "epoch": 36.4,
      "grad_norm": 0.6739000678062439,
      "kl": 0.3342580497264862,
      "learning_rate": 4.9971209662123774e-06,
      "loss": 0.0134,
      "reward": 1.24358332157135,
      "reward_std": 3.5022475719451904,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.5064166188240051,
      "rewards/wrapped_format_reward": 0.25,
      "step": 182
    },
    {
      "completion_length": 500.0,
      "epoch": 36.6,
      "grad_norm": 0.8527255654335022,
      "kl": 0.44380900263786316,
      "learning_rate": 4.996853343617542e-06,
      "loss": 0.0178,
      "reward": 1.3519909381866455,
      "reward_std": 2.9203834533691406,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.71875,
      "rewards/wrapped_driving_reward": -0.6167589426040649,
      "rewards/wrapped_format_reward": 0.5,
      "step": 183
    },
    {
      "completion_length": 500.0,
      "epoch": 36.8,
      "grad_norm": 0.6037353277206421,
      "kl": 0.3514931797981262,
      "learning_rate": 4.9965738368864345e-06,
      "loss": 0.0141,
      "reward": 2.4617958068847656,
      "reward_std": 0.43256813287734985,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.1632043421268463,
      "rewards/wrapped_format_reward": 0.625,
      "step": 184
    },
    {
      "completion_length": 500.0,
      "epoch": 37.0,
      "grad_norm": 0.6498645544052124,
      "kl": 0.39014145731925964,
      "learning_rate": 4.996282447349408e-06,
      "loss": 0.0156,
      "reward": 2.696049451828003,
      "reward_std": 0.6518055200576782,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.17895053327083588,
      "rewards/wrapped_format_reward": 0.875,
      "step": 185
    },
    {
      "completion_length": 500.0,
      "epoch": 37.2,
      "grad_norm": 0.6228243708610535,
      "kl": 0.2633248567581177,
      "learning_rate": 4.995979176393372e-06,
      "loss": 0.0105,
      "reward": 1.1363269090652466,
      "reward_std": 3.4644434452056885,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.9886730313301086,
      "rewards/wrapped_format_reward": 0.625,
      "step": 186
    },
    {
      "completion_length": 500.0,
      "epoch": 37.4,
      "grad_norm": 8.40079402923584,
      "kl": 1.8278297185897827,
      "learning_rate": 4.99566402546179e-06,
      "loss": 0.0731,
      "reward": -0.7244951725006104,
      "reward_std": 3.783473491668701,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.3705357313156128,
      "rewards/wrapped_driving_reward": -2.0950307846069336,
      "rewards/wrapped_format_reward": 0.5,
      "step": 187
    },
    {
      "completion_length": 500.0,
      "epoch": 37.6,
      "grad_norm": 0.5168763399124146,
      "kl": 0.2395801991224289,
      "learning_rate": 4.995336996054668e-06,
      "loss": 0.0096,
      "reward": 1.9002426862716675,
      "reward_std": 2.223823070526123,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9772727489471436,
      "rewards/wrapped_driving_reward": -0.7020300626754761,
      "rewards/wrapped_format_reward": 0.625,
      "step": 188
    },
    {
      "completion_length": 500.0,
      "epoch": 37.8,
      "grad_norm": 0.9863908290863037,
      "kl": 0.27976277470588684,
      "learning_rate": 4.99499808972855e-06,
      "loss": 0.0112,
      "reward": -0.028857052326202393,
      "reward_std": 2.8702406883239746,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -2.1538569927215576,
      "rewards/wrapped_format_reward": 0.625,
      "step": 189
    },
    {
      "completion_length": 500.0,
      "epoch": 38.0,
      "grad_norm": 0.8377166986465454,
      "kl": 0.48623228073120117,
      "learning_rate": 4.994647308096509e-06,
      "loss": 0.0194,
      "reward": 2.531177043914795,
      "reward_std": 0.5673744082450867,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9166666865348816,
      "rewards/wrapped_driving_reward": 0.4895104467868805,
      "rewards/wrapped_format_reward": 0.125,
      "step": 190
    },
    {
      "completion_length": 500.0,
      "epoch": 38.2,
      "grad_norm": 0.9249876737594604,
      "kl": 0.4526787996292114,
      "learning_rate": 4.994284652828143e-06,
      "loss": 0.0181,
      "reward": 0.6909130215644836,
      "reward_std": 3.1517491340637207,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.1840870380401611,
      "rewards/wrapped_format_reward": 0.375,
      "step": 191
    },
    {
      "completion_length": 500.0,
      "epoch": 38.4,
      "grad_norm": 0.5216014385223389,
      "kl": 0.2844958007335663,
      "learning_rate": 4.993910125649561e-06,
      "loss": 0.0114,
      "reward": 1.347219705581665,
      "reward_std": 3.583749771118164,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.527780294418335,
      "rewards/wrapped_format_reward": 0.375,
      "step": 192
    },
    {
      "completion_length": 500.0,
      "epoch": 38.6,
      "grad_norm": 0.7675309181213379,
      "kl": 0.46290096640586853,
      "learning_rate": 4.99352372834338e-06,
      "loss": 0.0185,
      "reward": 1.28756582736969,
      "reward_std": 3.200143337249756,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.7083333134651184,
      "rewards/wrapped_driving_reward": -0.5457674860954285,
      "rewards/wrapped_format_reward": 0.375,
      "step": 193
    },
    {
      "completion_length": 500.0,
      "epoch": 38.8,
      "grad_norm": 0.5267873406410217,
      "kl": 0.27246928215026855,
      "learning_rate": 4.993125462748714e-06,
      "loss": 0.0109,
      "reward": 0.5119737386703491,
      "reward_std": 2.572335958480835,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -1.9880262613296509,
      "rewards/wrapped_format_reward": 0.5,
      "step": 194
    },
    {
      "completion_length": 500.0,
      "epoch": 39.0,
      "grad_norm": 0.557345449924469,
      "kl": 0.33223679661750793,
      "learning_rate": 4.992715330761167e-06,
      "loss": 0.0133,
      "reward": 1.9005041122436523,
      "reward_std": 1.5405527353286743,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.5994958281517029,
      "rewards/wrapped_format_reward": 0.5,
      "step": 195
    },
    {
      "completion_length": 500.0,
      "epoch": 39.2,
      "grad_norm": 0.5145586729049683,
      "kl": 0.27872464060783386,
      "learning_rate": 4.992293334332821e-06,
      "loss": 0.0111,
      "reward": 0.08070141077041626,
      "reward_std": 2.161402702331543,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -2.9192986488342285,
      "rewards/wrapped_format_reward": 1.0,
      "step": 196
    },
    {
      "completion_length": 500.0,
      "epoch": 39.4,
      "grad_norm": 0.5731538534164429,
      "kl": 0.2947344481945038,
      "learning_rate": 4.9918594754722286e-06,
      "loss": 0.0118,
      "reward": 1.089212417602539,
      "reward_std": 3.5704760551452637,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.9107875823974609,
      "rewards/wrapped_format_reward": 0.5,
      "step": 197
    },
    {
      "completion_length": 500.0,
      "epoch": 39.6,
      "grad_norm": 1.0262069702148438,
      "kl": 0.36793074011802673,
      "learning_rate": 4.991413756244404e-06,
      "loss": 0.0147,
      "reward": 2.804293632507324,
      "reward_std": 0.05172164365649223,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.3042936325073242,
      "rewards/wrapped_format_reward": 0.5,
      "step": 198
    },
    {
      "completion_length": 500.0,
      "epoch": 39.8,
      "grad_norm": 0.7235340476036072,
      "kl": 0.4867457151412964,
      "learning_rate": 4.990956178770814e-06,
      "loss": 0.0195,
      "reward": 2.4924705028533936,
      "reward_std": 0.6009870767593384,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.949999988079071,
      "rewards/wrapped_driving_reward": -0.08252956718206406,
      "rewards/wrapped_format_reward": 0.625,
      "step": 199
    },
    {
      "completion_length": 500.0,
      "epoch": 40.0,
      "grad_norm": 0.8564599752426147,
      "kl": 0.4650922119617462,
      "learning_rate": 4.990486745229364e-06,
      "loss": 0.0186,
      "reward": 2.757322311401367,
      "reward_std": 0.5960695743560791,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9444444179534912,
      "rewards/wrapped_driving_reward": 0.18787765502929688,
      "rewards/wrapped_format_reward": 0.625,
      "step": 200
    },
    {
      "completion_length": 500.0,
      "epoch": 40.2,
      "grad_norm": 0.6181848645210266,
      "kl": 0.33555763959884644,
      "learning_rate": 4.990005457854392e-06,
      "loss": 0.0134,
      "reward": 0.935232937335968,
      "reward_std": 2.9882521629333496,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.7222222089767456,
      "rewards/wrapped_driving_reward": -0.7869893312454224,
      "rewards/wrapped_format_reward": 0.25,
      "step": 201
    },
    {
      "completion_length": 500.0,
      "epoch": 40.4,
      "grad_norm": 0.8061473369598389,
      "kl": 0.3526011109352112,
      "learning_rate": 4.989512318936654e-06,
      "loss": 0.0141,
      "reward": 2.038607597351074,
      "reward_std": 1.286082148551941,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9772727489471436,
      "rewards/wrapped_driving_reward": -0.4386652112007141,
      "rewards/wrapped_format_reward": 0.5,
      "step": 202
    },
    {
      "completion_length": 500.0,
      "epoch": 40.6,
      "grad_norm": 1.0745853185653687,
      "kl": 0.7225068807601929,
      "learning_rate": 4.989007330823319e-06,
      "loss": 0.0289,
      "reward": 3.327683210372925,
      "reward_std": 0.45302456617355347,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.5776832103729248,
      "rewards/wrapped_format_reward": 0.75,
      "step": 203
    },
    {
      "completion_length": 500.0,
      "epoch": 40.8,
      "grad_norm": 0.6797990202903748,
      "kl": 0.49457883834838867,
      "learning_rate": 4.988490495917948e-06,
      "loss": 0.0198,
      "reward": 1.4564661979675293,
      "reward_std": 3.6745243072509766,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.7935338020324707,
      "rewards/wrapped_format_reward": 0.75,
      "step": 204
    },
    {
      "completion_length": 500.0,
      "epoch": 41.0,
      "grad_norm": 0.5719887018203735,
      "kl": 0.3025702238082886,
      "learning_rate": 4.987961816680493e-06,
      "loss": 0.0121,
      "reward": 0.8813665509223938,
      "reward_std": 3.3135292530059814,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.685606062412262,
      "rewards/wrapped_driving_reward": -1.1792395114898682,
      "rewards/wrapped_format_reward": 0.625,
      "step": 205
    },
    {
      "completion_length": 500.0,
      "epoch": 41.2,
      "grad_norm": 0.7324315905570984,
      "kl": 0.387521356344223,
      "learning_rate": 4.987421295627279e-06,
      "loss": 0.0155,
      "reward": 3.60201358795166,
      "reward_std": 0.17326904833316803,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9722222089767456,
      "rewards/wrapped_driving_reward": 0.7547914981842041,
      "rewards/wrapped_format_reward": 0.875,
      "step": 206
    },
    {
      "completion_length": 500.0,
      "epoch": 41.4,
      "grad_norm": 1.4426076412200928,
      "kl": 0.3239262104034424,
      "learning_rate": 4.986868935330998e-06,
      "loss": 0.013,
      "reward": 1.1451337337493896,
      "reward_std": 3.175523042678833,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.9798662662506104,
      "rewards/wrapped_format_reward": 0.625,
      "step": 207
    },
    {
      "completion_length": 500.0,
      "epoch": 41.6,
      "grad_norm": 0.6265994310379028,
      "kl": 0.31086966395378113,
      "learning_rate": 4.986304738420684e-06,
      "loss": 0.0124,
      "reward": -0.08087223768234253,
      "reward_std": 3.9480772018432617,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.5808722972869873,
      "rewards/wrapped_format_reward": 0.5,
      "step": 208
    },
    {
      "completion_length": 500.0,
      "epoch": 41.8,
      "grad_norm": 0.5122293829917908,
      "kl": 0.22147461771965027,
      "learning_rate": 4.985728707581717e-06,
      "loss": 0.0089,
      "reward": 2.2255654335021973,
      "reward_std": 0.4417201578617096,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9821428656578064,
      "rewards/wrapped_driving_reward": -0.2565774619579315,
      "rewards/wrapped_format_reward": 0.5,
      "step": 209
    },
    {
      "completion_length": 500.0,
      "epoch": 42.0,
      "grad_norm": 0.5366212725639343,
      "kl": 0.2860429286956787,
      "learning_rate": 4.985140845555799e-06,
      "loss": 0.0114,
      "reward": -1.875,
      "reward_std": 1.108677864074707,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -4.0,
      "rewards/wrapped_format_reward": 0.625,
      "step": 210
    },
    {
      "completion_length": 500.0,
      "epoch": 42.2,
      "grad_norm": 0.757074773311615,
      "kl": 0.5041708946228027,
      "learning_rate": 4.984541155140945e-06,
      "loss": 0.0202,
      "reward": 1.3050158023834229,
      "reward_std": 3.2698206901550293,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.6428571343421936,
      "rewards/wrapped_driving_reward": -0.7128414511680603,
      "rewards/wrapped_format_reward": 0.625,
      "step": 211
    },
    {
      "completion_length": 500.0,
      "epoch": 42.4,
      "grad_norm": 0.5149911046028137,
      "kl": 0.24131189286708832,
      "learning_rate": 4.9839296391914696e-06,
      "loss": 0.0097,
      "reward": -0.5590072870254517,
      "reward_std": 3.6906325817108154,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.9340074062347412,
      "rewards/wrapped_format_reward": 0.375,
      "step": 212
    },
    {
      "completion_length": 500.0,
      "epoch": 42.6,
      "grad_norm": 0.7922428250312805,
      "kl": 0.4074100852012634,
      "learning_rate": 4.98330630061797e-06,
      "loss": 0.0163,
      "reward": 0.7251100540161133,
      "reward_std": 3.205897569656372,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.1498900651931763,
      "rewards/wrapped_format_reward": 0.375,
      "step": 213
    },
    {
      "completion_length": 500.0,
      "epoch": 42.8,
      "grad_norm": 0.8499237298965454,
      "kl": 0.533706784248352,
      "learning_rate": 4.982671142387316e-06,
      "loss": 0.0213,
      "reward": 1.2264912128448486,
      "reward_std": 3.1925883293151855,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.7735086679458618,
      "rewards/wrapped_format_reward": 0.5,
      "step": 214
    },
    {
      "completion_length": 500.0,
      "epoch": 43.0,
      "grad_norm": 0.5848891139030457,
      "kl": 0.4833756983280182,
      "learning_rate": 4.982024167522638e-06,
      "loss": 0.0193,
      "reward": 2.640871524810791,
      "reward_std": 0.3350675404071808,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.935606062412262,
      "rewards/wrapped_driving_reward": -0.1697344183921814,
      "rewards/wrapped_format_reward": 0.875,
      "step": 215
    },
    {
      "completion_length": 500.0,
      "epoch": 43.2,
      "grad_norm": 1.0190398693084717,
      "kl": 0.5212844014167786,
      "learning_rate": 4.981365379103306e-06,
      "loss": 0.0209,
      "reward": 1.518845796585083,
      "reward_std": 1.8981057405471802,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9772727489471436,
      "rewards/wrapped_driving_reward": -1.0834269523620605,
      "rewards/wrapped_format_reward": 0.625,
      "step": 216
    },
    {
      "completion_length": 500.0,
      "epoch": 43.4,
      "grad_norm": 0.6003134250640869,
      "kl": 0.2476293295621872,
      "learning_rate": 4.980694780264918e-06,
      "loss": 0.0099,
      "reward": 2.3462984561920166,
      "reward_std": 0.5958766937255859,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.4037014842033386,
      "rewards/wrapped_format_reward": 0.75,
      "step": 217
    },
    {
      "completion_length": 500.0,
      "epoch": 43.6,
      "grad_norm": 0.5352597832679749,
      "kl": 0.33760789036750793,
      "learning_rate": 4.980012374199288e-06,
      "loss": 0.0135,
      "reward": 1.1177078485488892,
      "reward_std": 3.422083854675293,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.8822920918464661,
      "rewards/wrapped_format_reward": 0.5,
      "step": 218
    },
    {
      "completion_length": 500.0,
      "epoch": 43.8,
      "grad_norm": 0.78425532579422,
      "kl": 0.45192739367485046,
      "learning_rate": 4.979318164154426e-06,
      "loss": 0.0181,
      "reward": 3.3331549167633057,
      "reward_std": 0.4030221104621887,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.8331548571586609,
      "rewards/wrapped_format_reward": 0.5,
      "step": 219
    },
    {
      "completion_length": 500.0,
      "epoch": 44.0,
      "grad_norm": 0.5511319041252136,
      "kl": 0.2625429630279541,
      "learning_rate": 4.978612153434527e-06,
      "loss": 0.0105,
      "reward": 3.4739222526550293,
      "reward_std": 0.35263335704803467,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.5989223122596741,
      "rewards/wrapped_format_reward": 0.875,
      "step": 220
    },
    {
      "completion_length": 500.0,
      "epoch": 44.2,
      "grad_norm": 0.8345232009887695,
      "kl": 0.5118071436882019,
      "learning_rate": 4.97789434539995e-06,
      "loss": 0.0205,
      "reward": 1.788142442703247,
      "reward_std": 2.3180289268493652,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -1.086857557296753,
      "rewards/wrapped_format_reward": 0.875,
      "step": 221
    },
    {
      "completion_length": 500.0,
      "epoch": 44.4,
      "grad_norm": 0.8292976021766663,
      "kl": 0.5234676003456116,
      "learning_rate": 4.977164743467206e-06,
      "loss": 0.0209,
      "reward": 1.3859682083129883,
      "reward_std": 3.6182608604431152,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.6875,
      "rewards/wrapped_driving_reward": -0.5515317916870117,
      "rewards/wrapped_format_reward": 0.5,
      "step": 222
    },
    {
      "completion_length": 500.0,
      "epoch": 44.6,
      "grad_norm": 0.8200549483299255,
      "kl": 0.3950418531894684,
      "learning_rate": 4.976423351108943e-06,
      "loss": 0.0158,
      "reward": 1.9203238487243652,
      "reward_std": 1.1563453674316406,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.7046762704849243,
      "rewards/wrapped_format_reward": 0.625,
      "step": 223
    },
    {
      "completion_length": 500.0,
      "epoch": 44.8,
      "grad_norm": 0.6968622207641602,
      "kl": 0.2271728217601776,
      "learning_rate": 4.975670171853926e-06,
      "loss": 0.0091,
      "reward": -0.3170052766799927,
      "reward_std": 2.1093220710754395,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9772727489471436,
      "rewards/wrapped_driving_reward": -2.919278144836426,
      "rewards/wrapped_format_reward": 0.625,
      "step": 224
    },
    {
      "completion_length": 500.0,
      "epoch": 45.0,
      "grad_norm": 0.7795050144195557,
      "kl": 0.4355601966381073,
      "learning_rate": 4.97490520928702e-06,
      "loss": 0.0174,
      "reward": 2.6324033737182617,
      "reward_std": 0.5314469933509827,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.11759641766548157,
      "rewards/wrapped_format_reward": 0.75,
      "step": 225
    },
    {
      "completion_length": 500.0,
      "epoch": 45.2,
      "grad_norm": 0.5524005889892578,
      "kl": 0.30952146649360657,
      "learning_rate": 4.974128467049177e-06,
      "loss": 0.0124,
      "reward": -2.125,
      "reward_std": 1.314977765083313,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -4.0,
      "rewards/wrapped_format_reward": 0.375,
      "step": 226
    },
    {
      "completion_length": 500.0,
      "epoch": 45.4,
      "grad_norm": 0.5645884871482849,
      "kl": 0.4939887821674347,
      "learning_rate": 4.9733399488374115e-06,
      "loss": 0.0198,
      "reward": 2.418989658355713,
      "reward_std": 0.14345024526119232,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.08101026713848114,
      "rewards/wrapped_format_reward": 0.5,
      "step": 227
    },
    {
      "completion_length": 500.0,
      "epoch": 45.6,
      "grad_norm": 0.9631263017654419,
      "kl": 0.6647568941116333,
      "learning_rate": 4.972539658404793e-06,
      "loss": 0.0266,
      "reward": -0.0228692889213562,
      "reward_std": 3.135000228881836,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -2.272869110107422,
      "rewards/wrapped_format_reward": 0.75,
      "step": 228
    },
    {
      "completion_length": 500.0,
      "epoch": 45.8,
      "grad_norm": 0.5868902802467346,
      "kl": 0.536701500415802,
      "learning_rate": 4.971727599560418e-06,
      "loss": 0.0215,
      "reward": 2.595135450363159,
      "reward_std": 0.5522119402885437,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.949999988079071,
      "rewards/wrapped_driving_reward": 0.020135482773184776,
      "rewards/wrapped_format_reward": 0.625,
      "step": 229
    },
    {
      "completion_length": 500.0,
      "epoch": 46.0,
      "grad_norm": 0.6927148103713989,
      "kl": 0.5391973257064819,
      "learning_rate": 4.970903776169403e-06,
      "loss": 0.0216,
      "reward": 3.2273426055908203,
      "reward_std": 0.38745206594467163,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9642857313156128,
      "rewards/wrapped_driving_reward": 0.7630569934844971,
      "rewards/wrapped_format_reward": 0.5,
      "step": 230
    },
    {
      "completion_length": 500.0,
      "epoch": 46.2,
      "grad_norm": 2.157358407974243,
      "kl": 0.5963761210441589,
      "learning_rate": 4.9700681921528495e-06,
      "loss": 0.0239,
      "reward": 3.3556950092315674,
      "reward_std": 0.5486971735954285,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.7306950092315674,
      "rewards/wrapped_format_reward": 0.625,
      "step": 231
    },
    {
      "completion_length": 500.0,
      "epoch": 46.4,
      "grad_norm": 0.5409197211265564,
      "kl": 0.31054040789604187,
      "learning_rate": 4.9692208514878445e-06,
      "loss": 0.0124,
      "reward": -1.75,
      "reward_std": 1.1902379989624023,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -4.0,
      "rewards/wrapped_format_reward": 0.75,
      "step": 232
    },
    {
      "completion_length": 500.0,
      "epoch": 46.6,
      "grad_norm": 0.8271388411521912,
      "kl": 0.5030784606933594,
      "learning_rate": 4.968361758207428e-06,
      "loss": 0.0201,
      "reward": 2.2951016426086426,
      "reward_std": 0.6324443817138672,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.07989836484193802,
      "rewards/wrapped_format_reward": 0.375,
      "step": 233
    },
    {
      "completion_length": 500.0,
      "epoch": 46.8,
      "grad_norm": 0.9013113975524902,
      "kl": 0.527148425579071,
      "learning_rate": 4.9674909164005805e-06,
      "loss": 0.0211,
      "reward": -0.08311975002288818,
      "reward_std": 4.243640422821045,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.5831197500228882,
      "rewards/wrapped_format_reward": 0.5,
      "step": 234
    },
    {
      "completion_length": 500.0,
      "epoch": 47.0,
      "grad_norm": 0.621760368347168,
      "kl": 0.5894174575805664,
      "learning_rate": 4.966608330212198e-06,
      "loss": 0.0236,
      "reward": 2.69521427154541,
      "reward_std": 0.2680894732475281,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9722222089767456,
      "rewards/wrapped_driving_reward": -0.15200814604759216,
      "rewards/wrapped_format_reward": 0.875,
      "step": 235
    },
    {
      "completion_length": 500.0,
      "epoch": 47.2,
      "grad_norm": 0.6673128604888916,
      "kl": 0.42412999272346497,
      "learning_rate": 4.965714003843079e-06,
      "loss": 0.017,
      "reward": -2.0,
      "reward_std": 1.0801234245300293,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -4.0,
      "rewards/wrapped_format_reward": 0.5,
      "step": 236
    },
    {
      "completion_length": 500.0,
      "epoch": 47.4,
      "grad_norm": 0.6826753616333008,
      "kl": 0.48437440395355225,
      "learning_rate": 4.9648079415499e-06,
      "loss": 0.0194,
      "reward": 2.6671550273895264,
      "reward_std": 0.6421502828598022,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.20784501731395721,
      "rewards/wrapped_format_reward": 0.875,
      "step": 237
    },
    {
      "completion_length": 500.0,
      "epoch": 47.6,
      "grad_norm": 0.7442097663879395,
      "kl": 0.5179538130760193,
      "learning_rate": 4.963890147645195e-06,
      "loss": 0.0207,
      "reward": 0.023519575595855713,
      "reward_std": 1.7913424968719482,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -2.351480484008789,
      "rewards/wrapped_format_reward": 0.375,
      "step": 238
    },
    {
      "completion_length": 500.0,
      "epoch": 47.8,
      "grad_norm": 0.9971833825111389,
      "kl": 0.2566893994808197,
      "learning_rate": 4.962960626497339e-06,
      "loss": 0.0103,
      "reward": 1.0741076469421387,
      "reward_std": 3.4465811252593994,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.6758923530578613,
      "rewards/wrapped_format_reward": 0.25,
      "step": 239
    },
    {
      "completion_length": 500.0,
      "epoch": 48.0,
      "grad_norm": 0.776371955871582,
      "kl": 0.6667019724845886,
      "learning_rate": 4.962019382530521e-06,
      "loss": 0.0267,
      "reward": 0.7681245803833008,
      "reward_std": 3.63140606880188,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.6068754196166992,
      "rewards/wrapped_format_reward": 0.875,
      "step": 240
    },
    {
      "completion_length": 500.0,
      "epoch": 48.2,
      "grad_norm": 0.958461344242096,
      "kl": 0.6015651226043701,
      "learning_rate": 4.961066420224729e-06,
      "loss": 0.0241,
      "reward": 0.8900174498558044,
      "reward_std": 2.1547889709472656,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9583333134651184,
      "rewards/wrapped_driving_reward": -1.568315863609314,
      "rewards/wrapped_format_reward": 0.5,
      "step": 241
    },
    {
      "completion_length": 500.0,
      "epoch": 48.4,
      "grad_norm": 0.8577614426612854,
      "kl": 0.7052382230758667,
      "learning_rate": 4.960101744115727e-06,
      "loss": 0.0282,
      "reward": 0.500007152557373,
      "reward_std": 3.6019463539123535,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.685606062412262,
      "rewards/wrapped_driving_reward": -1.6855988502502441,
      "rewards/wrapped_format_reward": 0.75,
      "step": 242
    },
    {
      "completion_length": 500.0,
      "epoch": 48.6,
      "grad_norm": 0.6088186502456665,
      "kl": 0.3410260081291199,
      "learning_rate": 4.959125358795031e-06,
      "loss": 0.0136,
      "reward": 1.2359226942062378,
      "reward_std": 3.157292127609253,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.1390773057937622,
      "rewards/wrapped_format_reward": 0.875,
      "step": 243
    },
    {
      "completion_length": 500.0,
      "epoch": 48.8,
      "grad_norm": 0.6780346035957336,
      "kl": 0.47339513897895813,
      "learning_rate": 4.958137268909887e-06,
      "loss": 0.0189,
      "reward": 1.3727295398712158,
      "reward_std": 3.2822999954223633,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.7250000238418579,
      "rewards/wrapped_driving_reward": -0.8522703647613525,
      "rewards/wrapped_format_reward": 0.75,
      "step": 244
    },
    {
      "completion_length": 500.0,
      "epoch": 49.0,
      "grad_norm": 0.6219626069068909,
      "kl": 0.3212871849536896,
      "learning_rate": 4.957137479163253e-06,
      "loss": 0.0129,
      "reward": 0.08353948593139648,
      "reward_std": 2.884551525115967,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -2.2914605140686035,
      "rewards/wrapped_format_reward": 0.875,
      "step": 245
    },
    {
      "completion_length": 500.0,
      "epoch": 49.2,
      "grad_norm": 0.8742188811302185,
      "kl": 0.6009516716003418,
      "learning_rate": 4.956125994313775e-06,
      "loss": 0.024,
      "reward": 3.219036817550659,
      "reward_std": 0.6377858519554138,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.8035714626312256,
      "rewards/wrapped_driving_reward": 0.5404652953147888,
      "rewards/wrapped_format_reward": 0.875,
      "step": 246
    },
    {
      "completion_length": 500.0,
      "epoch": 49.4,
      "grad_norm": 3.8272242546081543,
      "kl": 1.5439887046813965,
      "learning_rate": 4.95510281917576e-06,
      "loss": 0.0618,
      "reward": 3.679497241973877,
      "reward_std": 0.29719072580337524,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.6794970631599426,
      "rewards/wrapped_format_reward": 1.0,
      "step": 247
    },
    {
      "completion_length": 500.0,
      "epoch": 49.6,
      "grad_norm": 0.5449193120002747,
      "kl": 0.3074452579021454,
      "learning_rate": 4.9540679586191605e-06,
      "loss": 0.0123,
      "reward": -0.8099073171615601,
      "reward_std": 2.768624782562256,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -2.9349074363708496,
      "rewards/wrapped_format_reward": 0.625,
      "step": 248
    },
    {
      "completion_length": 500.0,
      "epoch": 49.8,
      "grad_norm": 0.9541939496994019,
      "kl": 0.5022038221359253,
      "learning_rate": 4.953021417569545e-06,
      "loss": 0.0201,
      "reward": 0.9676476120948792,
      "reward_std": 3.3370866775512695,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.1573524475097656,
      "rewards/wrapped_format_reward": 0.625,
      "step": 249
    },
    {
      "completion_length": 500.0,
      "epoch": 50.0,
      "grad_norm": 0.6913716197013855,
      "kl": 0.22377586364746094,
      "learning_rate": 4.9519632010080765e-06,
      "loss": 0.009,
      "reward": -0.7356908917427063,
      "reward_std": 3.197190761566162,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.4856908321380615,
      "rewards/wrapped_format_reward": 0.75,
      "step": 250
    }
  ],
  "logging_steps": 1,
  "max_steps": 1600,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 320,
  "save_steps": 250,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}