{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 100.0,
  "eval_steps": 500,
  "global_step": 500,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "completion_length": 500.0,
      "epoch": 0.2,
      "grad_norm": 144.23707580566406,
      "kl": 51.48179244995117,
      "learning_rate": 3.1250000000000005e-08,
      "loss": 2.0593,
      "reward": 0.9761996865272522,
      "reward_std": 3.3251326084136963,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.148800253868103,
      "rewards/wrapped_format_reward": 0.625,
      "step": 1
    },
    {
      "completion_length": 500.0,
      "epoch": 0.4,
      "grad_norm": 974.2139892578125,
      "kl": 216.24957275390625,
      "learning_rate": 6.250000000000001e-08,
      "loss": 8.65,
      "reward": -3.75,
      "reward_std": 0.5,
      "rewards/mpc_param_extraction_reward": 0.0,
      "rewards/mpc_param_name_reward": 0.0,
      "rewards/wrapped_driving_reward": -4.0,
      "rewards/wrapped_format_reward": 0.25,
      "step": 2
    },
    {
      "completion_length": 500.0,
      "epoch": 0.6,
      "grad_norm": 41697.91015625,
      "kl": 3837.41943359375,
      "learning_rate": 9.375e-08,
      "loss": 153.4967,
      "reward": -0.7961921691894531,
      "reward_std": 3.700653076171875,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.4375,
      "rewards/wrapped_driving_reward": -1.7336921691894531,
      "rewards/wrapped_format_reward": 0.0,
      "step": 3
    },
    {
      "completion_length": 500.0,
      "epoch": 0.8,
      "grad_norm": 10122959.0,
      "kl": 511094.90625,
      "learning_rate": 1.2500000000000002e-07,
      "loss": 20443.7988,
      "reward": -2.338921546936035,
      "reward_std": 3.322157144546509,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -2.838921546936035,
      "rewards/wrapped_format_reward": 0.0,
      "step": 4
    },
    {
      "completion_length": 500.0,
      "epoch": 1.0,
      "grad_norm": 100702232.0,
      "kl": 5416315.0,
      "learning_rate": 1.5625e-07,
      "loss": 216652.5938,
      "reward": -0.16450506448745728,
      "reward_std": 3.8515079021453857,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.6645050048828125,
      "rewards/wrapped_format_reward": 0.5,
      "step": 5
    },
    {
      "completion_length": 500.0,
      "epoch": 1.2,
      "grad_norm": 17.33243751525879,
      "kl": 7.672175884246826,
      "learning_rate": 1.875e-07,
      "loss": 0.3069,
      "reward": 0.9893605709075928,
      "reward_std": 1.5257619619369507,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -1.3856394290924072,
      "rewards/wrapped_format_reward": 0.375,
      "step": 6
    },
    {
      "completion_length": 500.0,
      "epoch": 1.4,
      "grad_norm": 70.34513092041016,
      "kl": 17.917146682739258,
      "learning_rate": 2.1875e-07,
      "loss": 0.7167,
      "reward": 1.2267450094223022,
      "reward_std": 3.4932949542999268,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.5232549905776978,
      "rewards/wrapped_format_reward": 0.25,
      "step": 7
    },
    {
      "completion_length": 500.0,
      "epoch": 1.6,
      "grad_norm": 69448.7734375,
      "kl": 9786.2802734375,
      "learning_rate": 2.5000000000000004e-07,
      "loss": 391.4512,
      "reward": -0.967779815196991,
      "reward_std": 3.5180184841156006,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.2177798748016357,
      "rewards/wrapped_format_reward": 0.25,
      "step": 8
    },
    {
      "completion_length": 500.0,
      "epoch": 1.8,
      "grad_norm": 7205431.5,
      "kl": 363326.15625,
      "learning_rate": 2.8125e-07,
      "loss": 14533.0439,
      "reward": -0.4434952139854431,
      "reward_std": 4.112156867980957,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.5684951543807983,
      "rewards/wrapped_format_reward": 0.125,
      "step": 9
    },
    {
      "completion_length": 500.0,
      "epoch": 2.0,
      "grad_norm": 1344.39306640625,
      "kl": 182.57179260253906,
      "learning_rate": 3.125e-07,
      "loss": 7.3029,
      "reward": -0.5283111929893494,
      "reward_std": 3.7256903648376465,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.6533112525939941,
      "rewards/wrapped_format_reward": 0.125,
      "step": 10
    },
    {
      "completion_length": 500.0,
      "epoch": 2.2,
      "grad_norm": 663.510498046875,
      "kl": 125.65680694580078,
      "learning_rate": 3.4375000000000004e-07,
      "loss": 5.0263,
      "reward": -2.449397563934326,
      "reward_std": 3.1012051105499268,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -2.949397563934326,
      "rewards/wrapped_format_reward": 0.0,
      "step": 11
    },
    {
      "completion_length": 500.0,
      "epoch": 2.4,
      "grad_norm": 31.331226348876953,
      "kl": 10.755382537841797,
      "learning_rate": 3.75e-07,
      "loss": 0.4302,
      "reward": -2.1561226844787598,
      "reward_std": 3.36269211769104,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -2.7811226844787598,
      "rewards/wrapped_format_reward": 0.125,
      "step": 12
    },
    {
      "completion_length": 500.0,
      "epoch": 2.6,
      "grad_norm": 10.003079414367676,
      "kl": 3.8625946044921875,
      "learning_rate": 4.0625000000000003e-07,
      "loss": 0.1545,
      "reward": -2.1760454177856445,
      "reward_std": 3.647908926010132,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -2.8010454177856445,
      "rewards/wrapped_format_reward": 0.125,
      "step": 13
    },
    {
      "completion_length": 500.0,
      "epoch": 2.8,
      "grad_norm": 62.2325325012207,
      "kl": 13.510702133178711,
      "learning_rate": 4.375e-07,
      "loss": 0.5404,
      "reward": -2.294018030166626,
      "reward_std": 3.0876402854919434,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -3.044018030166626,
      "rewards/wrapped_format_reward": 0.25,
      "step": 14
    },
    {
      "completion_length": 500.0,
      "epoch": 3.0,
      "grad_norm": 96.1074447631836,
      "kl": 10.679292678833008,
      "learning_rate": 4.6875000000000006e-07,
      "loss": 0.4272,
      "reward": -0.6371059417724609,
      "reward_std": 3.885227680206299,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.637105941772461,
      "rewards/wrapped_format_reward": 0.0,
      "step": 15
    },
    {
      "completion_length": 500.0,
      "epoch": 3.2,
      "grad_norm": 1650.8782958984375,
      "kl": 208.3596954345703,
      "learning_rate": 5.000000000000001e-07,
      "loss": 8.3344,
      "reward": -4.0,
      "reward_std": 0.0,
      "rewards/mpc_param_extraction_reward": 0.0,
      "rewards/mpc_param_name_reward": 0.0,
      "rewards/wrapped_driving_reward": -4.0,
      "rewards/wrapped_format_reward": 0.0,
      "step": 16
    },
    {
      "completion_length": 500.0,
      "epoch": 3.4,
      "grad_norm": 17.093393325805664,
      "kl": 5.905396461486816,
      "learning_rate": 5.3125e-07,
      "loss": 0.2362,
      "reward": -2.4352118968963623,
      "reward_std": 2.806159257888794,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -3.060211658477783,
      "rewards/wrapped_format_reward": 0.125,
      "step": 17
    },
    {
      "completion_length": 500.0,
      "epoch": 3.6,
      "grad_norm": 78087.4140625,
      "kl": 7675.3564453125,
      "learning_rate": 5.625e-07,
      "loss": 307.0142,
      "reward": -0.4786604046821594,
      "reward_std": 4.071903228759766,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.8536603450775146,
      "rewards/wrapped_format_reward": 0.375,
      "step": 18
    },
    {
      "completion_length": 500.0,
      "epoch": 3.8,
      "grad_norm": 2062.067626953125,
      "kl": 105.56303405761719,
      "learning_rate": 5.9375e-07,
      "loss": 4.2225,
      "reward": -0.4283701777458191,
      "reward_std": 4.145442008972168,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.8033702373504639,
      "rewards/wrapped_format_reward": 0.375,
      "step": 19
    },
    {
      "completion_length": 500.0,
      "epoch": 4.0,
      "grad_norm": 513992.65625,
      "kl": 39077.08984375,
      "learning_rate": 6.25e-07,
      "loss": 1563.0836,
      "reward": -2.0327651500701904,
      "reward_std": 3.3016297817230225,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -2.7827651500701904,
      "rewards/wrapped_format_reward": 0.25,
      "step": 20
    },
    {
      "completion_length": 500.0,
      "epoch": 4.2,
      "grad_norm": 271.5398254394531,
      "kl": 37.08869934082031,
      "learning_rate": 6.562500000000001e-07,
      "loss": 1.4835,
      "reward": -3.5,
      "reward_std": 0.5773502588272095,
      "rewards/mpc_param_extraction_reward": 0.0,
      "rewards/mpc_param_name_reward": 0.0,
      "rewards/wrapped_driving_reward": -4.0,
      "rewards/wrapped_format_reward": 0.5,
      "step": 21
    },
    {
      "completion_length": 500.0,
      "epoch": 4.4,
      "grad_norm": 33774.453125,
      "kl": 4115.1591796875,
      "learning_rate": 6.875000000000001e-07,
      "loss": 164.6064,
      "reward": -1.1355788707733154,
      "reward_std": 3.42315673828125,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.5105788707733154,
      "rewards/wrapped_format_reward": 0.375,
      "step": 22
    },
    {
      "completion_length": 500.0,
      "epoch": 4.6,
      "grad_norm": 52.09832000732422,
      "kl": 14.069100379943848,
      "learning_rate": 7.1875e-07,
      "loss": 0.5628,
      "reward": 0.9047523736953735,
      "reward_std": 3.2798702716827393,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.5952475666999817,
      "rewards/wrapped_format_reward": 0.0,
      "step": 23
    },
    {
      "completion_length": 500.0,
      "epoch": 4.8,
      "grad_norm": 75.83870697021484,
      "kl": 16.262989044189453,
      "learning_rate": 7.5e-07,
      "loss": 0.6505,
      "reward": -0.5572073459625244,
      "reward_std": 2.9710958003997803,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -2.0572073459625244,
      "rewards/wrapped_format_reward": 0.0,
      "step": 24
    },
    {
      "completion_length": 500.0,
      "epoch": 5.0,
      "grad_norm": 8.57257080078125,
      "kl": 3.3865182399749756,
      "learning_rate": 7.8125e-07,
      "loss": 0.1355,
      "reward": -2.110412120819092,
      "reward_std": 3.7791755199432373,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -2.860412120819092,
      "rewards/wrapped_format_reward": 0.25,
      "step": 25
    },
    {
      "completion_length": 500.0,
      "epoch": 5.2,
      "grad_norm": 978403.0,
      "kl": 89647.2578125,
      "learning_rate": 8.125000000000001e-07,
      "loss": 3585.8899,
      "reward": -0.2944529056549072,
      "reward_std": 4.2804718017578125,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.6694529056549072,
      "rewards/wrapped_format_reward": 0.375,
      "step": 26
    },
    {
      "completion_length": 500.0,
      "epoch": 5.4,
      "grad_norm": 168.73036193847656,
      "kl": 29.724079132080078,
      "learning_rate": 8.437500000000001e-07,
      "loss": 1.189,
      "reward": -0.6913368701934814,
      "reward_std": 3.5795211791992188,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.0663368701934814,
      "rewards/wrapped_format_reward": 0.375,
      "step": 27
    },
    {
      "completion_length": 500.0,
      "epoch": 5.6,
      "grad_norm": 75324.5234375,
      "kl": 7936.74267578125,
      "learning_rate": 8.75e-07,
      "loss": 317.4697,
      "reward": 0.9355948567390442,
      "reward_std": 3.3464736938476562,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.6894051432609558,
      "rewards/wrapped_format_reward": 0.125,
      "step": 28
    },
    {
      "completion_length": 500.0,
      "epoch": 5.8,
      "grad_norm": 469.9671630859375,
      "kl": 71.2878189086914,
      "learning_rate": 9.0625e-07,
      "loss": 2.8515,
      "reward": -2.547950267791748,
      "reward_std": 2.904099464416504,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -3.047950267791748,
      "rewards/wrapped_format_reward": 0.0,
      "step": 29
    },
    {
      "completion_length": 500.0,
      "epoch": 6.0,
      "grad_norm": 36.652645111083984,
      "kl": 13.148932456970215,
      "learning_rate": 9.375000000000001e-07,
      "loss": 0.526,
      "reward": -3.375,
      "reward_std": 0.9464846849441528,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -4.0,
      "rewards/wrapped_format_reward": 0.125,
      "step": 30
    },
    {
      "completion_length": 500.0,
      "epoch": 6.2,
      "grad_norm": 30.199684143066406,
      "kl": 9.480849266052246,
      "learning_rate": 9.6875e-07,
      "loss": 0.3792,
      "reward": -2.0530290603637695,
      "reward_std": 3.2615222930908203,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -2.9280290603637695,
      "rewards/wrapped_format_reward": 0.375,
      "step": 31
    },
    {
      "completion_length": 500.0,
      "epoch": 6.4,
      "grad_norm": 78.3298568725586,
      "kl": 26.2161865234375,
      "learning_rate": 1.0000000000000002e-06,
      "loss": 1.0486,
      "reward": 1.33430814743042,
      "reward_std": 3.5583572387695312,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.41569197177886963,
      "rewards/wrapped_format_reward": 0.25,
      "step": 32
    },
    {
      "completion_length": 500.0,
      "epoch": 6.6,
      "grad_norm": 19.472774505615234,
      "kl": 7.02009391784668,
      "learning_rate": 1.03125e-06,
      "loss": 0.2808,
      "reward": -0.6657888889312744,
      "reward_std": 3.853987216949463,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.6657888889312744,
      "rewards/wrapped_format_reward": 0.0,
      "step": 33
    },
    {
      "completion_length": 500.0,
      "epoch": 6.8,
      "grad_norm": 65.95396423339844,
      "kl": 18.14912223815918,
      "learning_rate": 1.0625e-06,
      "loss": 0.726,
      "reward": -0.8661626577377319,
      "reward_std": 3.6189870834350586,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.1161625385284424,
      "rewards/wrapped_format_reward": 0.25,
      "step": 34
    },
    {
      "completion_length": 500.0,
      "epoch": 7.0,
      "grad_norm": 1.4008393287658691,
      "kl": 0.8411699533462524,
      "learning_rate": 1.0937500000000001e-06,
      "loss": 0.0336,
      "reward": -0.38928359746932983,
      "reward_std": 4.171046257019043,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.6392836570739746,
      "rewards/wrapped_format_reward": 0.25,
      "step": 35
    },
    {
      "completion_length": 500.0,
      "epoch": 7.2,
      "grad_norm": 7.80991268157959,
      "kl": 4.218427658081055,
      "learning_rate": 1.125e-06,
      "loss": 0.1687,
      "reward": -1.1417465209960938,
      "reward_std": 3.381352186203003,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.1417465209960938,
      "rewards/wrapped_format_reward": 0.0,
      "step": 36
    },
    {
      "completion_length": 500.0,
      "epoch": 7.4,
      "grad_norm": 4.478097438812256,
      "kl": 2.112290143966675,
      "learning_rate": 1.1562500000000002e-06,
      "loss": 0.0845,
      "reward": -2.102426528930664,
      "reward_std": 3.1373467445373535,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -2.852426528930664,
      "rewards/wrapped_format_reward": 0.25,
      "step": 37
    },
    {
      "completion_length": 500.0,
      "epoch": 7.6,
      "grad_norm": 9.182758331298828,
      "kl": 4.63405179977417,
      "learning_rate": 1.1875e-06,
      "loss": 0.1854,
      "reward": -2.2051236629486084,
      "reward_std": 3.2649383544921875,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -2.9551236629486084,
      "rewards/wrapped_format_reward": 0.25,
      "step": 38
    },
    {
      "completion_length": 500.0,
      "epoch": 7.8,
      "grad_norm": 203072.515625,
      "kl": 20056.82421875,
      "learning_rate": 1.21875e-06,
      "loss": 802.2729,
      "reward": -2.125791311264038,
      "reward_std": 3.0907511711120605,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -3.000791311264038,
      "rewards/wrapped_format_reward": 0.375,
      "step": 39
    },
    {
      "completion_length": 500.0,
      "epoch": 8.0,
      "grad_norm": 5.843477725982666,
      "kl": 3.436691999435425,
      "learning_rate": 1.25e-06,
      "loss": 0.1375,
      "reward": -0.9351435899734497,
      "reward_std": 3.608586072921753,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.3101437091827393,
      "rewards/wrapped_format_reward": 0.375,
      "step": 40
    },
    {
      "completion_length": 500.0,
      "epoch": 8.2,
      "grad_norm": 7742784.5,
      "kl": 650933.375,
      "learning_rate": 1.28125e-06,
      "loss": 26037.334,
      "reward": 1.5518302917480469,
      "reward_std": 3.7290942668914795,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.4481697678565979,
      "rewards/wrapped_format_reward": 0.5,
      "step": 41
    },
    {
      "completion_length": 500.0,
      "epoch": 8.4,
      "grad_norm": 8.937725067138672,
      "kl": 3.0701639652252197,
      "learning_rate": 1.3125000000000001e-06,
      "loss": 0.1228,
      "reward": -2.1904397010803223,
      "reward_std": 3.2942306995391846,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -3.0654397010803223,
      "rewards/wrapped_format_reward": 0.375,
      "step": 42
    },
    {
      "completion_length": 500.0,
      "epoch": 8.6,
      "grad_norm": 14.36681079864502,
      "kl": 5.88793420791626,
      "learning_rate": 1.34375e-06,
      "loss": 0.2355,
      "reward": 0.6519123315811157,
      "reward_std": 3.105113983154297,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.973087728023529,
      "rewards/wrapped_format_reward": 0.125,
      "step": 43
    },
    {
      "completion_length": 500.0,
      "epoch": 8.8,
      "grad_norm": 180.1724395751953,
      "kl": 33.255760192871094,
      "learning_rate": 1.3750000000000002e-06,
      "loss": 1.3302,
      "reward": -0.6511552333831787,
      "reward_std": 3.9030916690826416,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.7761552333831787,
      "rewards/wrapped_format_reward": 0.125,
      "step": 44
    },
    {
      "completion_length": 500.0,
      "epoch": 9.0,
      "grad_norm": 32.81709671020508,
      "kl": 6.7791428565979,
      "learning_rate": 1.40625e-06,
      "loss": 0.2712,
      "reward": -3.875,
      "reward_std": 0.25,
      "rewards/mpc_param_extraction_reward": 0.0,
      "rewards/mpc_param_name_reward": 0.0,
      "rewards/wrapped_driving_reward": -4.0,
      "rewards/wrapped_format_reward": 0.125,
      "step": 45
    },
    {
      "completion_length": 500.0,
      "epoch": 9.2,
      "grad_norm": 4151.640625,
      "kl": 350.8572082519531,
      "learning_rate": 1.4375e-06,
      "loss": 14.0343,
      "reward": -1.1032943725585938,
      "reward_std": 3.3453028202056885,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.1032943725585938,
      "rewards/wrapped_format_reward": 0.0,
      "step": 46
    },
    {
      "completion_length": 500.0,
      "epoch": 9.4,
      "grad_norm": 15.409821510314941,
      "kl": 5.346187114715576,
      "learning_rate": 1.4687500000000001e-06,
      "loss": 0.2138,
      "reward": -1.1791430711746216,
      "reward_std": 2.8540005683898926,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -2.929143190383911,
      "rewards/wrapped_format_reward": 0.25,
      "step": 47
    },
    {
      "completion_length": 500.0,
      "epoch": 9.6,
      "grad_norm": 144.20443725585938,
      "kl": 23.608051300048828,
      "learning_rate": 1.5e-06,
      "loss": 0.9443,
      "reward": -2.530029058456421,
      "reward_std": 2.939941883087158,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.20000000298023224,
      "rewards/wrapped_driving_reward": -2.9800291061401367,
      "rewards/wrapped_format_reward": 0.0,
      "step": 48
    },
    {
      "completion_length": 500.0,
      "epoch": 9.8,
      "grad_norm": 7.547443866729736,
      "kl": 3.824962615966797,
      "learning_rate": 1.5312500000000002e-06,
      "loss": 0.153,
      "reward": -0.7816690802574158,
      "reward_std": 3.7201201915740967,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.906669020652771,
      "rewards/wrapped_format_reward": 0.125,
      "step": 49
    },
    {
      "completion_length": 500.0,
      "epoch": 10.0,
      "grad_norm": 77.945556640625,
      "kl": 16.699840545654297,
      "learning_rate": 1.5625e-06,
      "loss": 0.668,
      "reward": -0.2709696292877197,
      "reward_std": 4.022421360015869,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.6459696292877197,
      "rewards/wrapped_format_reward": 0.375,
      "step": 50
    },
    {
      "completion_length": 500.0,
      "epoch": 10.2,
      "grad_norm": 11738.6953125,
      "kl": 798.2957763671875,
      "learning_rate": 1.59375e-06,
      "loss": 31.9318,
      "reward": -2.244354724884033,
      "reward_std": 3.1866860389709473,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -2.869354724884033,
      "rewards/wrapped_format_reward": 0.125,
      "step": 51
    },
    {
      "completion_length": 500.0,
      "epoch": 10.4,
      "grad_norm": 14.563969612121582,
      "kl": 5.301497936248779,
      "learning_rate": 1.6250000000000001e-06,
      "loss": 0.2121,
      "reward": 2.282280445098877,
      "reward_std": 0.7978482246398926,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.21771956980228424,
      "rewards/wrapped_format_reward": 0.5,
      "step": 52
    },
    {
      "completion_length": 500.0,
      "epoch": 10.6,
      "grad_norm": 6.280083656311035,
      "kl": 3.3535187244415283,
      "learning_rate": 1.65625e-06,
      "loss": 0.1341,
      "reward": 0.2996126413345337,
      "reward_std": 2.949772357940674,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.3253873586654663,
      "rewards/wrapped_format_reward": 0.125,
      "step": 53
    },
    {
      "completion_length": 500.0,
      "epoch": 10.8,
      "grad_norm": 223484.046875,
      "kl": 25810.041015625,
      "learning_rate": 1.6875000000000001e-06,
      "loss": 1032.4015,
      "reward": -0.344623327255249,
      "reward_std": 3.944869041442871,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.719623327255249,
      "rewards/wrapped_format_reward": 0.375,
      "step": 54
    },
    {
      "completion_length": 500.0,
      "epoch": 11.0,
      "grad_norm": 593881.0,
      "kl": 99004.265625,
      "learning_rate": 1.71875e-06,
      "loss": 3960.1709,
      "reward": -3.875,
      "reward_std": 0.25,
      "rewards/mpc_param_extraction_reward": 0.0,
      "rewards/mpc_param_name_reward": 0.0,
      "rewards/wrapped_driving_reward": -4.0,
      "rewards/wrapped_format_reward": 0.125,
      "step": 55
    },
    {
      "completion_length": 500.0,
      "epoch": 11.2,
      "grad_norm": 6.934082508087158,
      "kl": 2.1647584438323975,
      "learning_rate": 1.75e-06,
      "loss": 0.0866,
      "reward": -1.2983622550964355,
      "reward_std": 2.1767022609710693,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -3.1733622550964355,
      "rewards/wrapped_format_reward": 0.375,
      "step": 56
    },
    {
      "completion_length": 500.0,
      "epoch": 11.4,
      "grad_norm": 11660.44921875,
      "kl": 1039.497802734375,
      "learning_rate": 1.78125e-06,
      "loss": 41.5799,
      "reward": 1.111867070198059,
      "reward_std": 3.4727301597595215,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.6381329298019409,
      "rewards/wrapped_format_reward": 0.25,
      "step": 57
    },
    {
      "completion_length": 500.0,
      "epoch": 11.6,
      "grad_norm": 116.61476135253906,
      "kl": 30.001558303833008,
      "learning_rate": 1.8125e-06,
      "loss": 1.2001,
      "reward": -3.875,
      "reward_std": 0.25,
      "rewards/mpc_param_extraction_reward": 0.0,
      "rewards/mpc_param_name_reward": 0.0,
      "rewards/wrapped_driving_reward": -4.0,
      "rewards/wrapped_format_reward": 0.125,
      "step": 58
    },
    {
      "completion_length": 500.0,
      "epoch": 11.8,
      "grad_norm": 7.750627040863037,
      "kl": 3.2049598693847656,
      "learning_rate": 1.8437500000000003e-06,
      "loss": 0.1282,
      "reward": -3.875,
      "reward_std": 0.25,
      "rewards/mpc_param_extraction_reward": 0.0,
      "rewards/mpc_param_name_reward": 0.0,
      "rewards/wrapped_driving_reward": -4.0,
      "rewards/wrapped_format_reward": 0.125,
      "step": 59
    },
    {
      "completion_length": 500.0,
      "epoch": 12.0,
      "grad_norm": 6.483101844787598,
      "kl": 2.8182482719421387,
      "learning_rate": 1.8750000000000003e-06,
      "loss": 0.1127,
      "reward": 0.4944196343421936,
      "reward_std": 2.8224055767059326,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.3805804252624512,
      "rewards/wrapped_format_reward": 0.375,
      "step": 60
    },
    {
      "completion_length": 500.0,
      "epoch": 12.2,
      "grad_norm": 2.41937255859375,
      "kl": 1.5243698358535767,
      "learning_rate": 1.90625e-06,
      "loss": 0.061,
      "reward": -1.5549238920211792,
      "reward_std": 3.3336286544799805,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.9299237728118896,
      "rewards/wrapped_format_reward": 0.375,
      "step": 61
    },
    {
      "completion_length": 500.0,
      "epoch": 12.4,
      "grad_norm": 1.8835395574569702,
      "kl": 1.3928029537200928,
      "learning_rate": 1.9375e-06,
      "loss": 0.0557,
      "reward": 0.5334538817405701,
      "reward_std": 3.0702548027038574,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.9665461778640747,
      "rewards/wrapped_format_reward": 0.0,
      "step": 62
    },
    {
      "completion_length": 500.0,
      "epoch": 12.6,
      "grad_norm": 82.16962432861328,
      "kl": 9.488784790039062,
      "learning_rate": 1.96875e-06,
      "loss": 0.3796,
      "reward": -0.4010847806930542,
      "reward_std": 4.155675411224365,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.7760847806930542,
      "rewards/wrapped_format_reward": 0.375,
      "step": 63
    },
    {
      "completion_length": 500.0,
      "epoch": 12.8,
      "grad_norm": 1.6330454349517822,
      "kl": 0.7770444750785828,
      "learning_rate": 2.0000000000000003e-06,
      "loss": 0.0311,
      "reward": -1.0365194082260132,
      "reward_std": 3.1399788856506348,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.46875,
      "rewards/wrapped_driving_reward": -2.2552695274353027,
      "rewards/wrapped_format_reward": 0.25,
      "step": 64
    },
    {
      "completion_length": 500.0,
      "epoch": 13.0,
      "grad_norm": 226.30535888671875,
      "kl": 45.10585021972656,
      "learning_rate": 2.0312500000000002e-06,
      "loss": 1.8042,
      "reward": -1.9248578548431396,
      "reward_std": 3.5153682231903076,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -2.7998578548431396,
      "rewards/wrapped_format_reward": 0.375,
      "step": 65
    },
    {
      "completion_length": 500.0,
      "epoch": 13.2,
      "grad_norm": 62.87789535522461,
      "kl": 7.880831718444824,
      "learning_rate": 2.0625e-06,
      "loss": 0.3152,
      "reward": 3.1509861946105957,
      "reward_std": 0.29935261607170105,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.987500011920929,
      "rewards/wrapped_driving_reward": 0.5384860038757324,
      "rewards/wrapped_format_reward": 0.625,
      "step": 66
    },
    {
      "completion_length": 500.0,
      "epoch": 13.4,
      "grad_norm": 41.82035446166992,
      "kl": 9.357061386108398,
      "learning_rate": 2.09375e-06,
      "loss": 0.3743,
      "reward": -2.576190948486328,
      "reward_std": 2.8476178646087646,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -3.201190948486328,
      "rewards/wrapped_format_reward": 0.125,
      "step": 67
    },
    {
      "completion_length": 500.0,
      "epoch": 13.6,
      "grad_norm": 4.275550365447998,
      "kl": 2.5297634601593018,
      "learning_rate": 2.125e-06,
      "loss": 0.1012,
      "reward": -2.036945104598999,
      "reward_std": 3.926109790802002,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -2.786945104598999,
      "rewards/wrapped_format_reward": 0.25,
      "step": 68
    },
    {
      "completion_length": 500.0,
      "epoch": 13.8,
      "grad_norm": 12.396137237548828,
      "kl": 3.0427801609039307,
      "learning_rate": 2.1562500000000003e-06,
      "loss": 0.1217,
      "reward": 0.7283755540847778,
      "reward_std": 3.323927879333496,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.2716244459152222,
      "rewards/wrapped_format_reward": 0.5,
      "step": 69
    },
    {
      "completion_length": 500.0,
      "epoch": 14.0,
      "grad_norm": 1.5282281637191772,
      "kl": 1.092595100402832,
      "learning_rate": 2.1875000000000002e-06,
      "loss": 0.0437,
      "reward": -3.2356114387512207,
      "reward_std": 1.528777003288269,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -3.8606114387512207,
      "rewards/wrapped_format_reward": 0.125,
      "step": 70
    },
    {
      "completion_length": 500.0,
      "epoch": 14.2,
      "grad_norm": 1.2480015754699707,
      "kl": 0.7834239602088928,
      "learning_rate": 2.21875e-06,
      "loss": 0.0313,
      "reward": -2.0355887413024902,
      "reward_std": 3.270659923553467,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -2.7855887413024902,
      "rewards/wrapped_format_reward": 0.25,
      "step": 71
    },
    {
      "completion_length": 500.0,
      "epoch": 14.4,
      "grad_norm": 80.85037231445312,
      "kl": 9.716327667236328,
      "learning_rate": 2.25e-06,
      "loss": 0.3887,
      "reward": -3.375,
      "reward_std": 1.25,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -4.0,
      "rewards/wrapped_format_reward": 0.125,
      "step": 72
    },
    {
      "completion_length": 500.0,
      "epoch": 14.6,
      "grad_norm": 1.1875276565551758,
      "kl": 0.9156450629234314,
      "learning_rate": 2.28125e-06,
      "loss": 0.0366,
      "reward": 0.398318886756897,
      "reward_std": 2.9533674716949463,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.3516812324523926,
      "rewards/wrapped_format_reward": 0.25,
      "step": 73
    },
    {
      "completion_length": 500.0,
      "epoch": 14.8,
      "grad_norm": 5.200348854064941,
      "kl": 2.256690502166748,
      "learning_rate": 2.3125000000000003e-06,
      "loss": 0.0903,
      "reward": -0.17332077026367188,
      "reward_std": 4.147373199462891,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.47727274894714355,
      "rewards/wrapped_driving_reward": -1.650593638420105,
      "rewards/wrapped_format_reward": 0.5,
      "step": 74
    },
    {
      "completion_length": 500.0,
      "epoch": 15.0,
      "grad_norm": 7008.42626953125,
      "kl": 815.8104248046875,
      "learning_rate": 2.3437500000000002e-06,
      "loss": 32.6324,
      "reward": -2.0281713008880615,
      "reward_std": 3.3107235431671143,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -2.9031713008880615,
      "rewards/wrapped_format_reward": 0.375,
      "step": 75
    },
    {
      "completion_length": 500.0,
      "epoch": 15.2,
      "grad_norm": 0.9020519852638245,
      "kl": 0.9176801443099976,
      "learning_rate": 2.375e-06,
      "loss": 0.0367,
      "reward": -2.1365058422088623,
      "reward_std": 3.0964157581329346,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -3.1365058422088623,
      "rewards/wrapped_format_reward": 0.5,
      "step": 76
    },
    {
      "completion_length": 500.0,
      "epoch": 15.4,
      "grad_norm": 2.2519280910491943,
      "kl": 0.8236393332481384,
      "learning_rate": 2.40625e-06,
      "loss": 0.0329,
      "reward": -1.22861909866333,
      "reward_std": 2.9241857528686523,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.353619337081909,
      "rewards/wrapped_format_reward": 0.125,
      "step": 77
    },
    {
      "completion_length": 500.0,
      "epoch": 15.6,
      "grad_norm": 1.5832031965255737,
      "kl": 0.7527546286582947,
      "learning_rate": 2.4375e-06,
      "loss": 0.0301,
      "reward": -0.6693365573883057,
      "reward_std": 3.860503911972046,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.7943366765975952,
      "rewards/wrapped_format_reward": 0.125,
      "step": 78
    },
    {
      "completion_length": 500.0,
      "epoch": 15.8,
      "grad_norm": 1.108726143836975,
      "kl": 1.0248883962631226,
      "learning_rate": 2.4687500000000003e-06,
      "loss": 0.041,
      "reward": -3.125,
      "reward_std": 1.4361406564712524,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -4.0,
      "rewards/wrapped_format_reward": 0.375,
      "step": 79
    },
    {
      "completion_length": 500.0,
      "epoch": 16.0,
      "grad_norm": 1.0169743299484253,
      "kl": 0.7592311501502991,
      "learning_rate": 2.5e-06,
      "loss": 0.0304,
      "reward": 1.145168423652649,
      "reward_std": 3.56965708732605,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.7298316359519958,
      "rewards/wrapped_format_reward": 0.375,
      "step": 80
    },
    {
      "completion_length": 500.0,
      "epoch": 16.2,
      "grad_norm": 0.7822604179382324,
      "kl": 0.560085117816925,
      "learning_rate": 2.53125e-06,
      "loss": 0.0224,
      "reward": 1.2056835889816284,
      "reward_std": 3.5178937911987305,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.6693164110183716,
      "rewards/wrapped_format_reward": 0.375,
      "step": 81
    },
    {
      "completion_length": 500.0,
      "epoch": 16.4,
      "grad_norm": 0.8175077438354492,
      "kl": 0.5752599239349365,
      "learning_rate": 2.5625e-06,
      "loss": 0.023,
      "reward": 2.9252407550811768,
      "reward_std": 0.7892647385597229,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.3002408742904663,
      "rewards/wrapped_format_reward": 0.625,
      "step": 82
    },
    {
      "completion_length": 500.0,
      "epoch": 16.6,
      "grad_norm": 6.47392463684082,
      "kl": 2.1055572032928467,
      "learning_rate": 2.5937500000000004e-06,
      "loss": 0.0842,
      "reward": -2.7708332538604736,
      "reward_std": 1.4678263664245605,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.4791666865348816,
      "rewards/wrapped_driving_reward": -4.0,
      "rewards/wrapped_format_reward": 0.25,
      "step": 83
    },
    {
      "completion_length": 500.0,
      "epoch": 16.8,
      "grad_norm": 0.9220647811889648,
      "kl": 0.7157851457595825,
      "learning_rate": 2.6250000000000003e-06,
      "loss": 0.0286,
      "reward": -2.987729549407959,
      "reward_std": 1.3781793117523193,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -3.987729549407959,
      "rewards/wrapped_format_reward": 0.5,
      "step": 84
    },
    {
      "completion_length": 500.0,
      "epoch": 17.0,
      "grad_norm": 62.959754943847656,
      "kl": 11.148348808288574,
      "learning_rate": 2.65625e-06,
      "loss": 0.4459,
      "reward": -3.875,
      "reward_std": 0.25,
      "rewards/mpc_param_extraction_reward": 0.0,
      "rewards/mpc_param_name_reward": 0.0,
      "rewards/wrapped_driving_reward": -4.0,
      "rewards/wrapped_format_reward": 0.125,
      "step": 85
    },
    {
      "completion_length": 500.0,
      "epoch": 17.2,
      "grad_norm": 1.0555181503295898,
      "kl": 0.7983899712562561,
      "learning_rate": 2.6875e-06,
      "loss": 0.0319,
      "reward": -0.34822893142700195,
      "reward_std": 1.868666648864746,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -2.973228931427002,
      "rewards/wrapped_format_reward": 0.625,
      "step": 86
    },
    {
      "completion_length": 500.0,
      "epoch": 17.4,
      "grad_norm": 5.494264602661133,
      "kl": 1.303008794784546,
      "learning_rate": 2.71875e-06,
      "loss": 0.0521,
      "reward": -1.6160635948181152,
      "reward_std": 3.132638931274414,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.7410635948181152,
      "rewards/wrapped_format_reward": 0.125,
      "step": 87
    },
    {
      "completion_length": 500.0,
      "epoch": 17.6,
      "grad_norm": 4.55032205581665,
      "kl": 0.8609241247177124,
      "learning_rate": 2.7500000000000004e-06,
      "loss": 0.0344,
      "reward": -0.07164722681045532,
      "reward_std": 2.4449591636657715,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.7045454978942871,
      "rewards/wrapped_driving_reward": -1.9011927843093872,
      "rewards/wrapped_format_reward": 0.375,
      "step": 88
    },
    {
      "completion_length": 500.0,
      "epoch": 17.8,
      "grad_norm": 0.9195135831832886,
      "kl": 0.683874785900116,
      "learning_rate": 2.7812500000000003e-06,
      "loss": 0.0274,
      "reward": -0.11890482902526855,
      "reward_std": 3.1548802852630615,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.699999988079071,
      "rewards/wrapped_driving_reward": -2.0689048767089844,
      "rewards/wrapped_format_reward": 0.5,
      "step": 89
    },
    {
      "completion_length": 500.0,
      "epoch": 18.0,
      "grad_norm": 33.38914108276367,
      "kl": 7.705580234527588,
      "learning_rate": 2.8125e-06,
      "loss": 0.3082,
      "reward": 1.586517572402954,
      "reward_std": 3.729795217514038,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.4134823679924011,
      "rewards/wrapped_format_reward": 0.5,
      "step": 90
    },
    {
      "completion_length": 500.0,
      "epoch": 18.2,
      "grad_norm": 57.7512092590332,
      "kl": 6.441009998321533,
      "learning_rate": 2.84375e-06,
      "loss": 0.2576,
      "reward": -0.8143091201782227,
      "reward_std": 3.7249650955200195,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.4444444477558136,
      "rewards/wrapped_driving_reward": -2.008753776550293,
      "rewards/wrapped_format_reward": 0.25,
      "step": 91
    },
    {
      "completion_length": 500.0,
      "epoch": 18.4,
      "grad_norm": 1.0778491497039795,
      "kl": 0.7857025265693665,
      "learning_rate": 2.875e-06,
      "loss": 0.0314,
      "reward": -0.4593994617462158,
      "reward_std": 3.805197238922119,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.8343994617462158,
      "rewards/wrapped_format_reward": 0.375,
      "step": 92
    },
    {
      "completion_length": 500.0,
      "epoch": 18.6,
      "grad_norm": 1.1437242031097412,
      "kl": 0.5162321925163269,
      "learning_rate": 2.9062500000000003e-06,
      "loss": 0.0206,
      "reward": -1.5748236179351807,
      "reward_std": 3.3948116302490234,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.8248236179351807,
      "rewards/wrapped_format_reward": 0.25,
      "step": 93
    },
    {
      "completion_length": 500.0,
      "epoch": 18.8,
      "grad_norm": 0.782410204410553,
      "kl": 0.4215336740016937,
      "learning_rate": 2.9375000000000003e-06,
      "loss": 0.0169,
      "reward": -1.7746977806091309,
      "reward_std": 3.3859715461730957,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.4375,
      "rewards/wrapped_driving_reward": -2.962197780609131,
      "rewards/wrapped_format_reward": 0.25,
      "step": 94
    },
    {
      "completion_length": 500.0,
      "epoch": 19.0,
      "grad_norm": 4.5705718994140625,
      "kl": 2.0152359008789062,
      "learning_rate": 2.96875e-06,
      "loss": 0.0806,
      "reward": 0.6353222131729126,
      "reward_std": 3.1809890270233154,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.1146776676177979,
      "rewards/wrapped_format_reward": 0.25,
      "step": 95
    },
    {
      "completion_length": 500.0,
      "epoch": 19.2,
      "grad_norm": 0.7173673510551453,
      "kl": 0.5042878985404968,
      "learning_rate": 3e-06,
      "loss": 0.0202,
      "reward": 3.3471288681030273,
      "reward_std": 0.31114432215690613,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.722128689289093,
      "rewards/wrapped_format_reward": 0.625,
      "step": 96
    },
    {
      "completion_length": 500.0,
      "epoch": 19.4,
      "grad_norm": 2.924496650695801,
      "kl": 0.9709882736206055,
      "learning_rate": 3.03125e-06,
      "loss": 0.0388,
      "reward": 1.2860008478164673,
      "reward_std": 2.1523053646087646,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9166666865348816,
      "rewards/wrapped_driving_reward": -1.1306657791137695,
      "rewards/wrapped_format_reward": 0.5,
      "step": 97
    },
    {
      "completion_length": 500.0,
      "epoch": 19.6,
      "grad_norm": 1.3449220657348633,
      "kl": 0.776192843914032,
      "learning_rate": 3.0625000000000003e-06,
      "loss": 0.031,
      "reward": -1.2475244998931885,
      "reward_std": 3.184887170791626,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.2475244998931885,
      "rewards/wrapped_format_reward": 0.0,
      "step": 98
    },
    {
      "completion_length": 500.0,
      "epoch": 19.8,
      "grad_norm": 16.095233917236328,
      "kl": 1.6037352085113525,
      "learning_rate": 3.0937500000000002e-06,
      "loss": 0.0641,
      "reward": 0.7092133164405823,
      "reward_std": 3.2424275875091553,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.040786862373352,
      "rewards/wrapped_format_reward": 0.25,
      "step": 99
    },
    {
      "completion_length": 500.0,
      "epoch": 20.0,
      "grad_norm": 1.065063714981079,
      "kl": 0.6967657208442688,
      "learning_rate": 3.125e-06,
      "loss": 0.0279,
      "reward": 1.58005690574646,
      "reward_std": 3.7297377586364746,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.41994309425354004,
      "rewards/wrapped_format_reward": 0.5,
      "step": 100
    },
    {
      "completion_length": 500.0,
      "epoch": 20.2,
      "grad_norm": 0.7442240715026855,
      "kl": 0.5057598352432251,
      "learning_rate": 3.15625e-06,
      "loss": 0.0202,
      "reward": -0.4889770746231079,
      "reward_std": 3.2432987689971924,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.1139769554138184,
      "rewards/wrapped_format_reward": 0.625,
      "step": 101
    },
    {
      "completion_length": 500.0,
      "epoch": 20.4,
      "grad_norm": 0.6366997361183167,
      "kl": 0.44367504119873047,
      "learning_rate": 3.1875e-06,
      "loss": 0.0177,
      "reward": -2.3750693798065186,
      "reward_std": 2.5939254760742188,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -3.2500693798065186,
      "rewards/wrapped_format_reward": 0.375,
      "step": 102
    },
    {
      "completion_length": 500.0,
      "epoch": 20.6,
      "grad_norm": 2.0096611976623535,
      "kl": 0.4689376652240753,
      "learning_rate": 3.2187500000000003e-06,
      "loss": 0.0188,
      "reward": -0.5932518243789673,
      "reward_std": 3.942629814147949,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.9682518243789673,
      "rewards/wrapped_format_reward": 0.375,
      "step": 103
    },
    {
      "completion_length": 500.0,
      "epoch": 20.8,
      "grad_norm": 1.2003757953643799,
      "kl": 0.3688035309314728,
      "learning_rate": 3.2500000000000002e-06,
      "loss": 0.0148,
      "reward": 2.4348607063293457,
      "reward_std": 1.402535080909729,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.824999988079071,
      "rewards/wrapped_driving_reward": 0.1098608672618866,
      "rewards/wrapped_format_reward": 0.5,
      "step": 104
    },
    {
      "completion_length": 500.0,
      "epoch": 21.0,
      "grad_norm": 1.0951755046844482,
      "kl": 0.6130416393280029,
      "learning_rate": 3.28125e-06,
      "loss": 0.0245,
      "reward": 1.912153720855713,
      "reward_std": 2.661609649658203,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.4628463387489319,
      "rewards/wrapped_format_reward": 0.375,
      "step": 105
    },
    {
      "completion_length": 500.0,
      "epoch": 21.2,
      "grad_norm": 0.7108362317085266,
      "kl": 0.4603574275970459,
      "learning_rate": 3.3125e-06,
      "loss": 0.0184,
      "reward": -0.09991639852523804,
      "reward_std": 2.701847791671753,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.699999988079071,
      "rewards/wrapped_driving_reward": -1.674916386604309,
      "rewards/wrapped_format_reward": 0.125,
      "step": 106
    },
    {
      "completion_length": 500.0,
      "epoch": 21.4,
      "grad_norm": 0.6701599955558777,
      "kl": 0.40188899636268616,
      "learning_rate": 3.34375e-06,
      "loss": 0.0161,
      "reward": -0.8211934566497803,
      "reward_std": 3.6706011295318604,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.0711934566497803,
      "rewards/wrapped_format_reward": 0.25,
      "step": 107
    },
    {
      "completion_length": 500.0,
      "epoch": 21.6,
      "grad_norm": 2.1547489166259766,
      "kl": 1.2839192152023315,
      "learning_rate": 3.3750000000000003e-06,
      "loss": 0.0514,
      "reward": 1.2595562934875488,
      "reward_std": 3.514036178588867,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.36544373631477356,
      "rewards/wrapped_format_reward": 0.125,
      "step": 108
    },
    {
      "completion_length": 500.0,
      "epoch": 21.8,
      "grad_norm": 1.838152527809143,
      "kl": 0.49252963066101074,
      "learning_rate": 3.40625e-06,
      "loss": 0.0197,
      "reward": 1.507678508758545,
      "reward_std": 3.675663948059082,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.7272727489471436,
      "rewards/wrapped_driving_reward": -0.46959418058395386,
      "rewards/wrapped_format_reward": 0.5,
      "step": 109
    },
    {
      "completion_length": 500.0,
      "epoch": 22.0,
      "grad_norm": 0.7439582347869873,
      "kl": 0.595367431640625,
      "learning_rate": 3.4375e-06,
      "loss": 0.0238,
      "reward": -3.625,
      "reward_std": 0.4787135720252991,
      "rewards/mpc_param_extraction_reward": 0.0,
      "rewards/mpc_param_name_reward": 0.0,
      "rewards/wrapped_driving_reward": -4.0,
      "rewards/wrapped_format_reward": 0.375,
      "step": 110
    },
    {
      "completion_length": 500.0,
      "epoch": 22.2,
      "grad_norm": 0.5864555835723877,
      "kl": 0.3803044855594635,
      "learning_rate": 3.46875e-06,
      "loss": 0.0152,
      "reward": 1.1622142791748047,
      "reward_std": 3.198068618774414,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.7272727489471436,
      "rewards/wrapped_driving_reward": -1.0650583505630493,
      "rewards/wrapped_format_reward": 0.75,
      "step": 111
    },
    {
      "completion_length": 500.0,
      "epoch": 22.4,
      "grad_norm": 4.526273250579834,
      "kl": 0.46392467617988586,
      "learning_rate": 3.5e-06,
      "loss": 0.0186,
      "reward": -0.7247750163078308,
      "reward_std": 3.514815330505371,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.0997748374938965,
      "rewards/wrapped_format_reward": 0.375,
      "step": 112
    },
    {
      "completion_length": 500.0,
      "epoch": 22.6,
      "grad_norm": 0.5704318881034851,
      "kl": 0.31563645601272583,
      "learning_rate": 3.5312500000000007e-06,
      "loss": 0.0126,
      "reward": -0.6594128608703613,
      "reward_std": 3.574946880340576,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.0344128608703613,
      "rewards/wrapped_format_reward": 0.375,
      "step": 113
    },
    {
      "completion_length": 500.0,
      "epoch": 22.8,
      "grad_norm": 0.671708881855011,
      "kl": 0.44867807626724243,
      "learning_rate": 3.5625e-06,
      "loss": 0.0179,
      "reward": 1.6787878274917603,
      "reward_std": 0.9073445200920105,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.9462121725082397,
      "rewards/wrapped_format_reward": 0.625,
      "step": 114
    },
    {
      "completion_length": 500.0,
      "epoch": 23.0,
      "grad_norm": 1.2352200746536255,
      "kl": 0.46100977063179016,
      "learning_rate": 3.59375e-06,
      "loss": 0.0184,
      "reward": -0.30863749980926514,
      "reward_std": 3.9797427654266357,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.8086374998092651,
      "rewards/wrapped_format_reward": 0.5,
      "step": 115
    },
    {
      "completion_length": 500.0,
      "epoch": 23.2,
      "grad_norm": 0.783157229423523,
      "kl": 0.44468817114830017,
      "learning_rate": 3.625e-06,
      "loss": 0.0178,
      "reward": 1.9851404428482056,
      "reward_std": 1.0498998165130615,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.2648596167564392,
      "rewards/wrapped_format_reward": 0.25,
      "step": 116
    },
    {
      "completion_length": 500.0,
      "epoch": 23.4,
      "grad_norm": 0.6096097826957703,
      "kl": 0.36140069365501404,
      "learning_rate": 3.65625e-06,
      "loss": 0.0145,
      "reward": -0.9730753898620605,
      "reward_std": 2.724126100540161,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -3.0980753898620605,
      "rewards/wrapped_format_reward": 0.625,
      "step": 117
    },
    {
      "completion_length": 500.0,
      "epoch": 23.6,
      "grad_norm": 2.4817147254943848,
      "kl": 0.3356289267539978,
      "learning_rate": 3.6875000000000007e-06,
      "loss": 0.0134,
      "reward": 2.9043874740600586,
      "reward_std": 0.34505343437194824,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.5293872952461243,
      "rewards/wrapped_format_reward": 0.375,
      "step": 118
    },
    {
      "completion_length": 500.0,
      "epoch": 23.8,
      "grad_norm": 1.3457905054092407,
      "kl": 0.32610735297203064,
      "learning_rate": 3.7187500000000006e-06,
      "loss": 0.013,
      "reward": -0.004844188690185547,
      "reward_std": 3.559382915496826,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.7548441886901855,
      "rewards/wrapped_format_reward": 0.25,
      "step": 119
    },
    {
      "completion_length": 500.0,
      "epoch": 24.0,
      "grad_norm": 0.8271002769470215,
      "kl": 0.592341423034668,
      "learning_rate": 3.7500000000000005e-06,
      "loss": 0.0237,
      "reward": 0.23737984895706177,
      "reward_std": 2.8921873569488525,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.6000000238418579,
      "rewards/wrapped_driving_reward": -1.2376201152801514,
      "rewards/wrapped_format_reward": 0.125,
      "step": 120
    },
    {
      "completion_length": 500.0,
      "epoch": 24.2,
      "grad_norm": 1.575377106666565,
      "kl": 0.31468361616134644,
      "learning_rate": 3.78125e-06,
      "loss": 0.0126,
      "reward": 0.08798408508300781,
      "reward_std": 3.364243984222412,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.9120157957077026,
      "rewards/wrapped_format_reward": 0.5,
      "step": 121
    },
    {
      "completion_length": 500.0,
      "epoch": 24.4,
      "grad_norm": 1.548542857170105,
      "kl": 0.7125066518783569,
      "learning_rate": 3.8125e-06,
      "loss": 0.0285,
      "reward": 3.202035903930664,
      "reward_std": 0.5515704154968262,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.8958333134651184,
      "rewards/wrapped_driving_reward": 0.6812027096748352,
      "rewards/wrapped_format_reward": 0.625,
      "step": 122
    },
    {
      "completion_length": 500.0,
      "epoch": 24.6,
      "grad_norm": 0.6466585397720337,
      "kl": 0.33141595125198364,
      "learning_rate": 3.84375e-06,
      "loss": 0.0133,
      "reward": -0.8563422560691833,
      "reward_std": 2.9308738708496094,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -2.981342315673828,
      "rewards/wrapped_format_reward": 0.625,
      "step": 123
    },
    {
      "completion_length": 500.0,
      "epoch": 24.8,
      "grad_norm": 0.9053751826286316,
      "kl": 0.3941192626953125,
      "learning_rate": 3.875e-06,
      "loss": 0.0158,
      "reward": -0.9088470935821533,
      "reward_std": 2.4116313457489014,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -2.6588470935821533,
      "rewards/wrapped_format_reward": 0.25,
      "step": 124
    },
    {
      "completion_length": 500.0,
      "epoch": 25.0,
      "grad_norm": 0.7404253482818604,
      "kl": 0.3537856936454773,
      "learning_rate": 3.90625e-06,
      "loss": 0.0142,
      "reward": -0.08935052156448364,
      "reward_std": 4.237273693084717,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.5893504619598389,
      "rewards/wrapped_format_reward": 0.5,
      "step": 125
    },
    {
      "completion_length": 500.0,
      "epoch": 25.2,
      "grad_norm": 0.5974608659744263,
      "kl": 0.31292691826820374,
      "learning_rate": 3.9375e-06,
      "loss": 0.0125,
      "reward": -0.5100458860397339,
      "reward_std": 3.746746063232422,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.8850458860397339,
      "rewards/wrapped_format_reward": 0.375,
      "step": 126
    },
    {
      "completion_length": 500.0,
      "epoch": 25.4,
      "grad_norm": 0.9886866807937622,
      "kl": 0.3266676068305969,
      "learning_rate": 3.96875e-06,
      "loss": 0.0131,
      "reward": 3.5397558212280273,
      "reward_std": 0.24529722332954407,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.7897558212280273,
      "rewards/wrapped_format_reward": 0.75,
      "step": 127
    },
    {
      "completion_length": 500.0,
      "epoch": 25.6,
      "grad_norm": 0.6569087505340576,
      "kl": 0.28314509987831116,
      "learning_rate": 4.000000000000001e-06,
      "loss": 0.0113,
      "reward": -0.5303106904029846,
      "reward_std": 4.0666303634643555,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.375,
      "rewards/wrapped_driving_reward": -1.7803106307983398,
      "rewards/wrapped_format_reward": 0.375,
      "step": 128
    },
    {
      "completion_length": 500.0,
      "epoch": 25.8,
      "grad_norm": 1.4679771661758423,
      "kl": 0.4160246253013611,
      "learning_rate": 4.031250000000001e-06,
      "loss": 0.0166,
      "reward": -0.5868573188781738,
      "reward_std": 3.941485643386841,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.8368571996688843,
      "rewards/wrapped_format_reward": 0.25,
      "step": 129
    },
    {
      "completion_length": 500.0,
      "epoch": 26.0,
      "grad_norm": 0.5985941290855408,
      "kl": 0.31736528873443604,
      "learning_rate": 4.0625000000000005e-06,
      "loss": 0.0127,
      "reward": 1.1113789081573486,
      "reward_std": 3.429651975631714,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.0136209726333618,
      "rewards/wrapped_format_reward": 0.625,
      "step": 130
    },
    {
      "completion_length": 500.0,
      "epoch": 26.2,
      "grad_norm": 0.6327362656593323,
      "kl": 0.40226221084594727,
      "learning_rate": 4.09375e-06,
      "loss": 0.0161,
      "reward": 0.7944153547286987,
      "reward_std": 2.8826069831848145,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.7250000238418579,
      "rewards/wrapped_driving_reward": -1.3055846691131592,
      "rewards/wrapped_format_reward": 0.625,
      "step": 131
    },
    {
      "completion_length": 500.0,
      "epoch": 26.4,
      "grad_norm": 0.6797881722450256,
      "kl": 0.4582635164260864,
      "learning_rate": 4.125e-06,
      "loss": 0.0183,
      "reward": 2.8031327724456787,
      "reward_std": 0.7006269097328186,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.05313277989625931,
      "rewards/wrapped_format_reward": 0.75,
      "step": 132
    },
    {
      "completion_length": 500.0,
      "epoch": 26.6,
      "grad_norm": 0.5752917528152466,
      "kl": 0.36153456568717957,
      "learning_rate": 4.15625e-06,
      "loss": 0.0145,
      "reward": -2.564105987548828,
      "reward_std": 2.871788263320923,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -3.189105987548828,
      "rewards/wrapped_format_reward": 0.125,
      "step": 133
    },
    {
      "completion_length": 500.0,
      "epoch": 26.8,
      "grad_norm": 0.569823145866394,
      "kl": 0.3600581884384155,
      "learning_rate": 4.1875e-06,
      "loss": 0.0144,
      "reward": 3.2037861347198486,
      "reward_std": 0.1732039451599121,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.8287861943244934,
      "rewards/wrapped_format_reward": 0.375,
      "step": 134
    },
    {
      "completion_length": 500.0,
      "epoch": 27.0,
      "grad_norm": 11.942618370056152,
      "kl": 2.177290678024292,
      "learning_rate": 4.21875e-06,
      "loss": 0.0871,
      "reward": -2.3714582920074463,
      "reward_std": 1.8921570777893066,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -3.7464582920074463,
      "rewards/wrapped_format_reward": 0.375,
      "step": 135
    },
    {
      "completion_length": 500.0,
      "epoch": 27.2,
      "grad_norm": 0.5660642385482788,
      "kl": 0.2908819019794464,
      "learning_rate": 4.25e-06,
      "loss": 0.0116,
      "reward": -0.6192033290863037,
      "reward_std": 3.6331570148468018,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.1192033290863037,
      "rewards/wrapped_format_reward": 0.5,
      "step": 136
    },
    {
      "completion_length": 500.0,
      "epoch": 27.4,
      "grad_norm": 1.4041975736618042,
      "kl": 0.463067889213562,
      "learning_rate": 4.28125e-06,
      "loss": 0.0185,
      "reward": -2.75,
      "reward_std": 1.1902379989624023,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -4.0,
      "rewards/wrapped_format_reward": 0.25,
      "step": 137
    },
    {
      "completion_length": 500.0,
      "epoch": 27.6,
      "grad_norm": 0.4801470637321472,
      "kl": 0.2532914876937866,
      "learning_rate": 4.312500000000001e-06,
      "loss": 0.0101,
      "reward": -2.304798126220703,
      "reward_std": 3.3904037475585938,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -3.054798126220703,
      "rewards/wrapped_format_reward": 0.25,
      "step": 138
    },
    {
      "completion_length": 500.0,
      "epoch": 27.8,
      "grad_norm": 0.6999854445457458,
      "kl": 0.4938638210296631,
      "learning_rate": 4.3437500000000006e-06,
      "loss": 0.0198,
      "reward": -1.9556548595428467,
      "reward_std": 3.430131196975708,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -2.9556548595428467,
      "rewards/wrapped_format_reward": 0.5,
      "step": 139
    },
    {
      "completion_length": 500.0,
      "epoch": 28.0,
      "grad_norm": 1.7622352838516235,
      "kl": 0.32535520195961,
      "learning_rate": 4.3750000000000005e-06,
      "loss": 0.013,
      "reward": 3.048956871032715,
      "reward_std": 0.7497459053993225,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.42395687103271484,
      "rewards/wrapped_format_reward": 0.625,
      "step": 140
    },
    {
      "completion_length": 500.0,
      "epoch": 28.2,
      "grad_norm": 1.0910435914993286,
      "kl": 0.3166691064834595,
      "learning_rate": 4.40625e-06,
      "loss": 0.0127,
      "reward": 2.1717541217803955,
      "reward_std": 2.45133900642395,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.45324593782424927,
      "rewards/wrapped_format_reward": 0.625,
      "step": 141
    },
    {
      "completion_length": 500.0,
      "epoch": 28.4,
      "grad_norm": 0.563035249710083,
      "kl": 0.34334975481033325,
      "learning_rate": 4.4375e-06,
      "loss": 0.0137,
      "reward": -0.30545544624328613,
      "reward_std": 2.531362295150757,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.71875,
      "rewards/wrapped_driving_reward": -2.149205446243286,
      "rewards/wrapped_format_reward": 0.375,
      "step": 142
    },
    {
      "completion_length": 500.0,
      "epoch": 28.6,
      "grad_norm": 0.6513370871543884,
      "kl": 0.2893451154232025,
      "learning_rate": 4.46875e-06,
      "loss": 0.0116,
      "reward": -0.7655331492424011,
      "reward_std": 3.818908214569092,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.265533208847046,
      "rewards/wrapped_format_reward": 0.5,
      "step": 143
    },
    {
      "completion_length": 500.0,
      "epoch": 28.8,
      "grad_norm": 0.6747258305549622,
      "kl": 0.4012701213359833,
      "learning_rate": 4.5e-06,
      "loss": 0.0161,
      "reward": -1.631712794303894,
      "reward_std": 2.7382092475891113,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -3.2567129135131836,
      "rewards/wrapped_format_reward": 0.625,
      "step": 144
    },
    {
      "completion_length": 500.0,
      "epoch": 29.0,
      "grad_norm": 1.388415813446045,
      "kl": 0.3030587136745453,
      "learning_rate": 4.53125e-06,
      "loss": 0.0121,
      "reward": -0.3709021210670471,
      "reward_std": 3.0999691486358643,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.6666666865348816,
      "rewards/wrapped_driving_reward": -2.0375688076019287,
      "rewards/wrapped_format_reward": 0.25,
      "step": 145
    },
    {
      "completion_length": 500.0,
      "epoch": 29.2,
      "grad_norm": 1.3835958242416382,
      "kl": 0.5185285806655884,
      "learning_rate": 4.5625e-06,
      "loss": 0.0207,
      "reward": -0.25881457328796387,
      "reward_std": 2.5411531925201416,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9010416865348816,
      "rewards/wrapped_driving_reward": -2.6598563194274902,
      "rewards/wrapped_format_reward": 0.5,
      "step": 146
    },
    {
      "completion_length": 500.0,
      "epoch": 29.4,
      "grad_norm": 1.0529229640960693,
      "kl": 0.304034560918808,
      "learning_rate": 4.59375e-06,
      "loss": 0.0122,
      "reward": 0.5033270120620728,
      "reward_std": 3.8062596321105957,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.6216729879379272,
      "rewards/wrapped_format_reward": 0.625,
      "step": 147
    },
    {
      "completion_length": 500.0,
      "epoch": 29.6,
      "grad_norm": 1.3924496173858643,
      "kl": 0.3519279956817627,
      "learning_rate": 4.625000000000001e-06,
      "loss": 0.0141,
      "reward": 1.081155776977539,
      "reward_std": 2.07023286819458,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9750000238418579,
      "rewards/wrapped_driving_reward": -1.1438441276550293,
      "rewards/wrapped_format_reward": 0.25,
      "step": 148
    },
    {
      "completion_length": 500.0,
      "epoch": 29.8,
      "grad_norm": 1.6642379760742188,
      "kl": 0.5595217347145081,
      "learning_rate": 4.6562500000000005e-06,
      "loss": 0.0224,
      "reward": 2.879631519317627,
      "reward_std": 0.5703426003456116,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.12963154911994934,
      "rewards/wrapped_format_reward": 0.75,
      "step": 149
    },
    {
      "completion_length": 500.0,
      "epoch": 30.0,
      "grad_norm": 0.5775982737541199,
      "kl": 0.2810514271259308,
      "learning_rate": 4.6875000000000004e-06,
      "loss": 0.0112,
      "reward": 0.10444420576095581,
      "reward_std": 3.2514774799346924,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -2.0205557346343994,
      "rewards/wrapped_format_reward": 0.625,
      "step": 150
    },
    {
      "completion_length": 500.0,
      "epoch": 30.2,
      "grad_norm": 0.9198185801506042,
      "kl": 0.28956174850463867,
      "learning_rate": 4.71875e-06,
      "loss": 0.0116,
      "reward": -0.260436087846756,
      "reward_std": 2.8927173614501953,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -2.2604360580444336,
      "rewards/wrapped_format_reward": 0.5,
      "step": 151
    },
    {
      "completion_length": 500.0,
      "epoch": 30.4,
      "grad_norm": 0.7754166722297668,
      "kl": 0.38463443517684937,
      "learning_rate": 4.75e-06,
      "loss": 0.0154,
      "reward": 0.4493406414985657,
      "reward_std": 2.646808385848999,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.300659418106079,
      "rewards/wrapped_format_reward": 0.25,
      "step": 152
    },
    {
      "completion_length": 500.0,
      "epoch": 30.6,
      "grad_norm": 0.5780096650123596,
      "kl": 0.3385607898235321,
      "learning_rate": 4.781250000000001e-06,
      "loss": 0.0135,
      "reward": -1.7927734851837158,
      "reward_std": 3.755190849304199,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -2.792773485183716,
      "rewards/wrapped_format_reward": 0.5,
      "step": 153
    },
    {
      "completion_length": 500.0,
      "epoch": 30.8,
      "grad_norm": 0.5552729964256287,
      "kl": 0.28436288237571716,
      "learning_rate": 4.8125e-06,
      "loss": 0.0114,
      "reward": 0.6222386360168457,
      "reward_std": 2.1850173473358154,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -1.8777613639831543,
      "rewards/wrapped_format_reward": 0.5,
      "step": 154
    },
    {
      "completion_length": 500.0,
      "epoch": 31.0,
      "grad_norm": 0.9199939370155334,
      "kl": 0.37593454122543335,
      "learning_rate": 4.84375e-06,
      "loss": 0.015,
      "reward": 0.4306233525276184,
      "reward_std": 3.5992963314056396,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.6607142686843872,
      "rewards/wrapped_driving_reward": -1.605090856552124,
      "rewards/wrapped_format_reward": 0.625,
      "step": 155
    },
    {
      "completion_length": 500.0,
      "epoch": 31.2,
      "grad_norm": 0.5603945851325989,
      "kl": 0.3141997754573822,
      "learning_rate": 4.875e-06,
      "loss": 0.0126,
      "reward": -2.0,
      "reward_std": 1.3540064096450806,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -4.0,
      "rewards/wrapped_format_reward": 0.5,
      "step": 156
    },
    {
      "completion_length": 500.0,
      "epoch": 31.4,
      "grad_norm": 0.6190344095230103,
      "kl": 0.27537742257118225,
      "learning_rate": 4.90625e-06,
      "loss": 0.011,
      "reward": 1.0282058715820312,
      "reward_std": 3.394833564758301,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.846794068813324,
      "rewards/wrapped_format_reward": 0.375,
      "step": 157
    },
    {
      "completion_length": 500.0,
      "epoch": 31.6,
      "grad_norm": 0.6877399682998657,
      "kl": 0.2958383858203888,
      "learning_rate": 4.937500000000001e-06,
      "loss": 0.0118,
      "reward": -0.755041241645813,
      "reward_std": 3.4646472930908203,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.1300413608551025,
      "rewards/wrapped_format_reward": 0.375,
      "step": 158
    },
    {
      "completion_length": 500.0,
      "epoch": 31.8,
      "grad_norm": 31.595932006835938,
      "kl": 6.78364372253418,
      "learning_rate": 4.9687500000000005e-06,
      "loss": 0.2713,
      "reward": 0.35857605934143066,
      "reward_std": 2.9158554077148438,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.7664239406585693,
      "rewards/wrapped_format_reward": 0.625,
      "step": 159
    },
    {
      "completion_length": 500.0,
      "epoch": 32.0,
      "grad_norm": 1.7048529386520386,
      "kl": 0.35252463817596436,
      "learning_rate": 5e-06,
      "loss": 0.0141,
      "reward": 2.64233136177063,
      "reward_std": 0.7985239624977112,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.14233140647411346,
      "rewards/wrapped_format_reward": 0.5,
      "step": 160
    },
    {
      "completion_length": 500.0,
      "epoch": 32.2,
      "grad_norm": 0.9699507355690002,
      "kl": 0.3963090479373932,
      "learning_rate": 4.99999405044338e-06,
      "loss": 0.0159,
      "reward": 2.578547477722168,
      "reward_std": 0.16937123239040375,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.046452634036540985,
      "rewards/wrapped_format_reward": 0.625,
      "step": 161
    },
    {
      "completion_length": 500.0,
      "epoch": 32.4,
      "grad_norm": 0.6427643299102783,
      "kl": 0.2770542800426483,
      "learning_rate": 4.999976201801837e-06,
      "loss": 0.0111,
      "reward": 2.2058539390563965,
      "reward_std": 1.1022424697875977,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.41914597153663635,
      "rewards/wrapped_format_reward": 0.625,
      "step": 162
    },
    {
      "completion_length": 500.0,
      "epoch": 32.6,
      "grad_norm": 0.6896190047264099,
      "kl": 0.27440541982650757,
      "learning_rate": 4.999946454160323e-06,
      "loss": 0.011,
      "reward": 1.1675429344177246,
      "reward_std": 3.4587650299072266,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.7142857313156128,
      "rewards/wrapped_driving_reward": -0.7967426180839539,
      "rewards/wrapped_format_reward": 0.5,
      "step": 163
    },
    {
      "completion_length": 500.0,
      "epoch": 32.8,
      "grad_norm": 0.7695831060409546,
      "kl": 0.4198772609233856,
      "learning_rate": 4.9999048076604286e-06,
      "loss": 0.0168,
      "reward": -1.912316918373108,
      "reward_std": 2.555265426635742,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.9123167991638184,
      "rewards/wrapped_format_reward": 0.0,
      "step": 164
    },
    {
      "completion_length": 500.0,
      "epoch": 33.0,
      "grad_norm": 0.5920954942703247,
      "kl": 0.28969520330429077,
      "learning_rate": 4.999851262500375e-06,
      "loss": 0.0116,
      "reward": 3.1377110481262207,
      "reward_std": 0.5497580170631409,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.5127109289169312,
      "rewards/wrapped_format_reward": 0.625,
      "step": 165
    },
    {
      "completion_length": 500.0,
      "epoch": 33.2,
      "grad_norm": 6.077564716339111,
      "kl": 0.44747114181518555,
      "learning_rate": 4.999785818935018e-06,
      "loss": 0.0179,
      "reward": 2.047877073287964,
      "reward_std": 2.722182035446167,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.45212286710739136,
      "rewards/wrapped_format_reward": 0.5,
      "step": 166
    },
    {
      "completion_length": 500.0,
      "epoch": 33.4,
      "grad_norm": 0.6889002919197083,
      "kl": 0.37658053636550903,
      "learning_rate": 4.999708477275846e-06,
      "loss": 0.0151,
      "reward": -2.284590482711792,
      "reward_std": 3.106440782546997,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -3.159590482711792,
      "rewards/wrapped_format_reward": 0.375,
      "step": 167
    },
    {
      "completion_length": 500.0,
      "epoch": 33.6,
      "grad_norm": 1.8645473718643188,
      "kl": 0.3408987522125244,
      "learning_rate": 4.9996192378909785e-06,
      "loss": 0.0136,
      "reward": 0.917718231678009,
      "reward_std": 2.948974132537842,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.699999988079071,
      "rewards/wrapped_driving_reward": -1.157281756401062,
      "rewards/wrapped_format_reward": 0.625,
      "step": 168
    },
    {
      "completion_length": 500.0,
      "epoch": 33.8,
      "grad_norm": 0.535763680934906,
      "kl": 0.25453072786331177,
      "learning_rate": 4.999518101205162e-06,
      "loss": 0.0102,
      "reward": 3.604552745819092,
      "reward_std": 0.45598289370536804,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.7295528054237366,
      "rewards/wrapped_format_reward": 0.875,
      "step": 169
    },
    {
      "completion_length": 500.0,
      "epoch": 34.0,
      "grad_norm": 1.0853776931762695,
      "kl": 0.2871979773044586,
      "learning_rate": 4.999405067699773e-06,
      "loss": 0.0115,
      "reward": 0.7697337865829468,
      "reward_std": 3.0176069736480713,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.3552662134170532,
      "rewards/wrapped_format_reward": 0.625,
      "step": 170
    },
    {
      "completion_length": 500.0,
      "epoch": 34.2,
      "grad_norm": 2.175551176071167,
      "kl": 0.7303879261016846,
      "learning_rate": 4.99928013791281e-06,
      "loss": 0.0292,
      "reward": 0.010015249252319336,
      "reward_std": 4.346557140350342,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.46875,
      "rewards/wrapped_driving_reward": -1.5837347507476807,
      "rewards/wrapped_format_reward": 0.625,
      "step": 171
    },
    {
      "completion_length": 500.0,
      "epoch": 34.4,
      "grad_norm": 1.3378883600234985,
      "kl": 0.2555471658706665,
      "learning_rate": 4.999143312438893e-06,
      "loss": 0.0102,
      "reward": 1.064118504524231,
      "reward_std": 1.464298963546753,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -1.4358813762664795,
      "rewards/wrapped_format_reward": 0.5,
      "step": 172
    },
    {
      "completion_length": 500.0,
      "epoch": 34.6,
      "grad_norm": 1.6005758047103882,
      "kl": 0.3272940516471863,
      "learning_rate": 4.998994591929266e-06,
      "loss": 0.0131,
      "reward": 3.320277214050293,
      "reward_std": 0.5942137241363525,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.8202772736549377,
      "rewards/wrapped_format_reward": 0.5,
      "step": 173
    },
    {
      "completion_length": 500.0,
      "epoch": 34.8,
      "grad_norm": 0.8775622844696045,
      "kl": 0.3981474041938782,
      "learning_rate": 4.998833977091783e-06,
      "loss": 0.0159,
      "reward": 2.548191547393799,
      "reward_std": 0.13038182258605957,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.17319151759147644,
      "rewards/wrapped_format_reward": 0.375,
      "step": 174
    },
    {
      "completion_length": 500.0,
      "epoch": 35.0,
      "grad_norm": 0.5131356716156006,
      "kl": 0.26495081186294556,
      "learning_rate": 4.998661468690914e-06,
      "loss": 0.0106,
      "reward": 0.2881455421447754,
      "reward_std": 3.1594552993774414,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.7118544578552246,
      "rewards/wrapped_format_reward": 0.5,
      "step": 175
    },
    {
      "completion_length": 500.0,
      "epoch": 35.2,
      "grad_norm": 1.4990577697753906,
      "kl": 0.3656232953071594,
      "learning_rate": 4.99847706754774e-06,
      "loss": 0.0146,
      "reward": 2.0933961868286133,
      "reward_std": 0.39702948927879333,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9375,
      "rewards/wrapped_driving_reward": -0.594103991985321,
      "rewards/wrapped_format_reward": 0.75,
      "step": 176
    },
    {
      "completion_length": 500.0,
      "epoch": 35.4,
      "grad_norm": 0.5740483999252319,
      "kl": 0.265653520822525,
      "learning_rate": 4.998280774539943e-06,
      "loss": 0.0106,
      "reward": 1.1700050830841064,
      "reward_std": 3.1657402515411377,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.8299949765205383,
      "rewards/wrapped_format_reward": 0.5,
      "step": 177
    },
    {
      "completion_length": 500.0,
      "epoch": 35.6,
      "grad_norm": 0.6564896702766418,
      "kl": 0.265337198972702,
      "learning_rate": 4.998072590601808e-06,
      "loss": 0.0106,
      "reward": -0.852949857711792,
      "reward_std": 3.3822429180145264,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.102949857711792,
      "rewards/wrapped_format_reward": 0.25,
      "step": 178
    },
    {
      "completion_length": 500.0,
      "epoch": 35.8,
      "grad_norm": 23.83641242980957,
      "kl": 4.303451061248779,
      "learning_rate": 4.9978525167242176e-06,
      "loss": 0.1721,
      "reward": 0.764412522315979,
      "reward_std": 2.8684115409851074,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.7250000238418579,
      "rewards/wrapped_driving_reward": -1.085587501525879,
      "rewards/wrapped_format_reward": 0.375,
      "step": 179
    },
    {
      "completion_length": 500.0,
      "epoch": 36.0,
      "grad_norm": 0.7350974082946777,
      "kl": 0.30466321110725403,
      "learning_rate": 4.997620553954645e-06,
      "loss": 0.0122,
      "reward": -0.10997164249420166,
      "reward_std": 2.883012056350708,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -2.109971523284912,
      "rewards/wrapped_format_reward": 0.5,
      "step": 180
    },
    {
      "completion_length": 500.0,
      "epoch": 36.2,
      "grad_norm": 1.8978265523910522,
      "kl": 0.5050737857818604,
      "learning_rate": 4.997376703397151e-06,
      "loss": 0.0202,
      "reward": -0.35431569814682007,
      "reward_std": 4.209678649902344,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.8543156385421753,
      "rewards/wrapped_format_reward": 0.5,
      "step": 181
    },
    {
      "completion_length": 500.0,
      "epoch": 36.4,
      "grad_norm": 0.6739000678062439,
      "kl": 0.3342580497264862,
      "learning_rate": 4.9971209662123774e-06,
      "loss": 0.0134,
      "reward": 1.24358332157135,
      "reward_std": 3.5022475719451904,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.5064166188240051,
      "rewards/wrapped_format_reward": 0.25,
      "step": 182
    },
    {
      "completion_length": 500.0,
      "epoch": 36.6,
      "grad_norm": 0.8527255654335022,
      "kl": 0.44380900263786316,
      "learning_rate": 4.996853343617542e-06,
      "loss": 0.0178,
      "reward": 1.3519909381866455,
      "reward_std": 2.9203834533691406,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.71875,
      "rewards/wrapped_driving_reward": -0.6167589426040649,
      "rewards/wrapped_format_reward": 0.5,
      "step": 183
    },
    {
      "completion_length": 500.0,
      "epoch": 36.8,
      "grad_norm": 0.6037353277206421,
      "kl": 0.3514931797981262,
      "learning_rate": 4.9965738368864345e-06,
      "loss": 0.0141,
      "reward": 2.4617958068847656,
      "reward_std": 0.43256813287734985,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.1632043421268463,
      "rewards/wrapped_format_reward": 0.625,
      "step": 184
    },
    {
      "completion_length": 500.0,
      "epoch": 37.0,
      "grad_norm": 0.6498645544052124,
      "kl": 0.39014145731925964,
      "learning_rate": 4.996282447349408e-06,
      "loss": 0.0156,
      "reward": 2.696049451828003,
      "reward_std": 0.6518055200576782,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.17895053327083588,
      "rewards/wrapped_format_reward": 0.875,
      "step": 185
    },
    {
      "completion_length": 500.0,
      "epoch": 37.2,
      "grad_norm": 0.6228243708610535,
      "kl": 0.2633248567581177,
      "learning_rate": 4.995979176393372e-06,
      "loss": 0.0105,
      "reward": 1.1363269090652466,
      "reward_std": 3.4644434452056885,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.9886730313301086,
      "rewards/wrapped_format_reward": 0.625,
      "step": 186
    },
    {
      "completion_length": 500.0,
      "epoch": 37.4,
      "grad_norm": 8.40079402923584,
      "kl": 1.8278297185897827,
      "learning_rate": 4.99566402546179e-06,
      "loss": 0.0731,
      "reward": -0.7244951725006104,
      "reward_std": 3.783473491668701,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.3705357313156128,
      "rewards/wrapped_driving_reward": -2.0950307846069336,
      "rewards/wrapped_format_reward": 0.5,
      "step": 187
    },
    {
      "completion_length": 500.0,
      "epoch": 37.6,
      "grad_norm": 0.5168763399124146,
      "kl": 0.2395801991224289,
      "learning_rate": 4.995336996054668e-06,
      "loss": 0.0096,
      "reward": 1.9002426862716675,
      "reward_std": 2.223823070526123,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9772727489471436,
      "rewards/wrapped_driving_reward": -0.7020300626754761,
      "rewards/wrapped_format_reward": 0.625,
      "step": 188
    },
    {
      "completion_length": 500.0,
      "epoch": 37.8,
      "grad_norm": 0.9863908290863037,
      "kl": 0.27976277470588684,
      "learning_rate": 4.99499808972855e-06,
      "loss": 0.0112,
      "reward": -0.028857052326202393,
      "reward_std": 2.8702406883239746,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -2.1538569927215576,
      "rewards/wrapped_format_reward": 0.625,
      "step": 189
    },
    {
      "completion_length": 500.0,
      "epoch": 38.0,
      "grad_norm": 0.8377166986465454,
      "kl": 0.48623228073120117,
      "learning_rate": 4.994647308096509e-06,
      "loss": 0.0194,
      "reward": 2.531177043914795,
      "reward_std": 0.5673744082450867,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9166666865348816,
      "rewards/wrapped_driving_reward": 0.4895104467868805,
      "rewards/wrapped_format_reward": 0.125,
      "step": 190
    },
    {
      "completion_length": 500.0,
      "epoch": 38.2,
      "grad_norm": 0.9249876737594604,
      "kl": 0.4526787996292114,
      "learning_rate": 4.994284652828143e-06,
      "loss": 0.0181,
      "reward": 0.6909130215644836,
      "reward_std": 3.1517491340637207,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.1840870380401611,
      "rewards/wrapped_format_reward": 0.375,
      "step": 191
    },
    {
      "completion_length": 500.0,
      "epoch": 38.4,
      "grad_norm": 0.5216014385223389,
      "kl": 0.2844958007335663,
      "learning_rate": 4.993910125649561e-06,
      "loss": 0.0114,
      "reward": 1.347219705581665,
      "reward_std": 3.583749771118164,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.527780294418335,
      "rewards/wrapped_format_reward": 0.375,
      "step": 192
    },
    {
      "completion_length": 500.0,
      "epoch": 38.6,
      "grad_norm": 0.7675309181213379,
      "kl": 0.46290096640586853,
      "learning_rate": 4.99352372834338e-06,
      "loss": 0.0185,
      "reward": 1.28756582736969,
      "reward_std": 3.200143337249756,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.7083333134651184,
      "rewards/wrapped_driving_reward": -0.5457674860954285,
      "rewards/wrapped_format_reward": 0.375,
      "step": 193
    },
    {
      "completion_length": 500.0,
      "epoch": 38.8,
      "grad_norm": 0.5267873406410217,
      "kl": 0.27246928215026855,
      "learning_rate": 4.993125462748714e-06,
      "loss": 0.0109,
      "reward": 0.5119737386703491,
      "reward_std": 2.572335958480835,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -1.9880262613296509,
      "rewards/wrapped_format_reward": 0.5,
      "step": 194
    },
    {
      "completion_length": 500.0,
      "epoch": 39.0,
      "grad_norm": 0.557345449924469,
      "kl": 0.33223679661750793,
      "learning_rate": 4.992715330761167e-06,
      "loss": 0.0133,
      "reward": 1.9005041122436523,
      "reward_std": 1.5405527353286743,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.5994958281517029,
      "rewards/wrapped_format_reward": 0.5,
      "step": 195
    },
    {
      "completion_length": 500.0,
      "epoch": 39.2,
      "grad_norm": 0.5145586729049683,
      "kl": 0.27872464060783386,
      "learning_rate": 4.992293334332821e-06,
      "loss": 0.0111,
      "reward": 0.08070141077041626,
      "reward_std": 2.161402702331543,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -2.9192986488342285,
      "rewards/wrapped_format_reward": 1.0,
      "step": 196
    },
    {
      "completion_length": 500.0,
      "epoch": 39.4,
      "grad_norm": 0.5731538534164429,
      "kl": 0.2947344481945038,
      "learning_rate": 4.9918594754722286e-06,
      "loss": 0.0118,
      "reward": 1.089212417602539,
      "reward_std": 3.5704760551452637,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.9107875823974609,
      "rewards/wrapped_format_reward": 0.5,
      "step": 197
    },
    {
      "completion_length": 500.0,
      "epoch": 39.6,
      "grad_norm": 1.0262069702148438,
      "kl": 0.36793074011802673,
      "learning_rate": 4.991413756244404e-06,
      "loss": 0.0147,
      "reward": 2.804293632507324,
      "reward_std": 0.05172164365649223,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.3042936325073242,
      "rewards/wrapped_format_reward": 0.5,
      "step": 198
    },
    {
      "completion_length": 500.0,
      "epoch": 39.8,
      "grad_norm": 0.7235340476036072,
      "kl": 0.4867457151412964,
      "learning_rate": 4.990956178770814e-06,
      "loss": 0.0195,
      "reward": 2.4924705028533936,
      "reward_std": 0.6009870767593384,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.949999988079071,
      "rewards/wrapped_driving_reward": -0.08252956718206406,
      "rewards/wrapped_format_reward": 0.625,
      "step": 199
    },
    {
      "completion_length": 500.0,
      "epoch": 40.0,
      "grad_norm": 0.8564599752426147,
      "kl": 0.4650922119617462,
      "learning_rate": 4.990486745229364e-06,
      "loss": 0.0186,
      "reward": 2.757322311401367,
      "reward_std": 0.5960695743560791,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9444444179534912,
      "rewards/wrapped_driving_reward": 0.18787765502929688,
      "rewards/wrapped_format_reward": 0.625,
      "step": 200
    },
    {
      "completion_length": 500.0,
      "epoch": 40.2,
      "grad_norm": 0.6181848645210266,
      "kl": 0.33555763959884644,
      "learning_rate": 4.990005457854392e-06,
      "loss": 0.0134,
      "reward": 0.935232937335968,
      "reward_std": 2.9882521629333496,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.7222222089767456,
      "rewards/wrapped_driving_reward": -0.7869893312454224,
      "rewards/wrapped_format_reward": 0.25,
      "step": 201
    },
    {
      "completion_length": 500.0,
      "epoch": 40.4,
      "grad_norm": 0.8061473369598389,
      "kl": 0.3526011109352112,
      "learning_rate": 4.989512318936654e-06,
      "loss": 0.0141,
      "reward": 2.038607597351074,
      "reward_std": 1.286082148551941,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9772727489471436,
      "rewards/wrapped_driving_reward": -0.4386652112007141,
      "rewards/wrapped_format_reward": 0.5,
      "step": 202
    },
    {
      "completion_length": 500.0,
      "epoch": 40.6,
      "grad_norm": 1.0745853185653687,
      "kl": 0.7225068807601929,
      "learning_rate": 4.989007330823319e-06,
      "loss": 0.0289,
      "reward": 3.327683210372925,
      "reward_std": 0.45302456617355347,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.5776832103729248,
      "rewards/wrapped_format_reward": 0.75,
      "step": 203
    },
    {
      "completion_length": 500.0,
      "epoch": 40.8,
      "grad_norm": 0.6797990202903748,
      "kl": 0.49457883834838867,
      "learning_rate": 4.988490495917948e-06,
      "loss": 0.0198,
      "reward": 1.4564661979675293,
      "reward_std": 3.6745243072509766,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.7935338020324707,
      "rewards/wrapped_format_reward": 0.75,
      "step": 204
    },
    {
      "completion_length": 500.0,
      "epoch": 41.0,
      "grad_norm": 0.5719887018203735,
      "kl": 0.3025702238082886,
      "learning_rate": 4.987961816680493e-06,
      "loss": 0.0121,
      "reward": 0.8813665509223938,
      "reward_std": 3.3135292530059814,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.685606062412262,
      "rewards/wrapped_driving_reward": -1.1792395114898682,
      "rewards/wrapped_format_reward": 0.625,
      "step": 205
    },
    {
      "completion_length": 500.0,
      "epoch": 41.2,
      "grad_norm": 0.7324315905570984,
      "kl": 0.387521356344223,
      "learning_rate": 4.987421295627279e-06,
      "loss": 0.0155,
      "reward": 3.60201358795166,
      "reward_std": 0.17326904833316803,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9722222089767456,
      "rewards/wrapped_driving_reward": 0.7547914981842041,
      "rewards/wrapped_format_reward": 0.875,
      "step": 206
    },
    {
      "completion_length": 500.0,
      "epoch": 41.4,
      "grad_norm": 1.4426076412200928,
      "kl": 0.3239262104034424,
      "learning_rate": 4.986868935330998e-06,
      "loss": 0.013,
      "reward": 1.1451337337493896,
      "reward_std": 3.175523042678833,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.9798662662506104,
      "rewards/wrapped_format_reward": 0.625,
      "step": 207
    },
    {
      "completion_length": 500.0,
      "epoch": 41.6,
      "grad_norm": 0.6265994310379028,
      "kl": 0.31086966395378113,
      "learning_rate": 4.986304738420684e-06,
      "loss": 0.0124,
      "reward": -0.08087223768234253,
      "reward_std": 3.9480772018432617,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.5808722972869873,
      "rewards/wrapped_format_reward": 0.5,
      "step": 208
    },
    {
      "completion_length": 500.0,
      "epoch": 41.8,
      "grad_norm": 0.5122293829917908,
      "kl": 0.22147461771965027,
      "learning_rate": 4.985728707581717e-06,
      "loss": 0.0089,
      "reward": 2.2255654335021973,
      "reward_std": 0.4417201578617096,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9821428656578064,
      "rewards/wrapped_driving_reward": -0.2565774619579315,
      "rewards/wrapped_format_reward": 0.5,
      "step": 209
    },
    {
      "completion_length": 500.0,
      "epoch": 42.0,
      "grad_norm": 0.5366212725639343,
      "kl": 0.2860429286956787,
      "learning_rate": 4.985140845555799e-06,
      "loss": 0.0114,
      "reward": -1.875,
      "reward_std": 1.108677864074707,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -4.0,
      "rewards/wrapped_format_reward": 0.625,
      "step": 210
    },
    {
      "completion_length": 500.0,
      "epoch": 42.2,
      "grad_norm": 0.757074773311615,
      "kl": 0.5041708946228027,
      "learning_rate": 4.984541155140945e-06,
      "loss": 0.0202,
      "reward": 1.3050158023834229,
      "reward_std": 3.2698206901550293,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.6428571343421936,
      "rewards/wrapped_driving_reward": -0.7128414511680603,
      "rewards/wrapped_format_reward": 0.625,
      "step": 211
    },
    {
      "completion_length": 500.0,
      "epoch": 42.4,
      "grad_norm": 0.5149911046028137,
      "kl": 0.24131189286708832,
      "learning_rate": 4.9839296391914696e-06,
      "loss": 0.0097,
      "reward": -0.5590072870254517,
      "reward_std": 3.6906325817108154,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.9340074062347412,
      "rewards/wrapped_format_reward": 0.375,
      "step": 212
    },
    {
      "completion_length": 500.0,
      "epoch": 42.6,
      "grad_norm": 0.7922428250312805,
      "kl": 0.4074100852012634,
      "learning_rate": 4.98330630061797e-06,
      "loss": 0.0163,
      "reward": 0.7251100540161133,
      "reward_std": 3.205897569656372,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.1498900651931763,
      "rewards/wrapped_format_reward": 0.375,
      "step": 213
    },
    {
      "completion_length": 500.0,
      "epoch": 42.8,
      "grad_norm": 0.8499237298965454,
      "kl": 0.533706784248352,
      "learning_rate": 4.982671142387316e-06,
      "loss": 0.0213,
      "reward": 1.2264912128448486,
      "reward_std": 3.1925883293151855,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.7735086679458618,
      "rewards/wrapped_format_reward": 0.5,
      "step": 214
    },
    {
      "completion_length": 500.0,
      "epoch": 43.0,
      "grad_norm": 0.5848891139030457,
      "kl": 0.4833756983280182,
      "learning_rate": 4.982024167522638e-06,
      "loss": 0.0193,
      "reward": 2.640871524810791,
      "reward_std": 0.3350675404071808,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.935606062412262,
      "rewards/wrapped_driving_reward": -0.1697344183921814,
      "rewards/wrapped_format_reward": 0.875,
      "step": 215
    },
    {
      "completion_length": 500.0,
      "epoch": 43.2,
      "grad_norm": 1.0190398693084717,
      "kl": 0.5212844014167786,
      "learning_rate": 4.981365379103306e-06,
      "loss": 0.0209,
      "reward": 1.518845796585083,
      "reward_std": 1.8981057405471802,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9772727489471436,
      "rewards/wrapped_driving_reward": -1.0834269523620605,
      "rewards/wrapped_format_reward": 0.625,
      "step": 216
    },
    {
      "completion_length": 500.0,
      "epoch": 43.4,
      "grad_norm": 0.6003134250640869,
      "kl": 0.2476293295621872,
      "learning_rate": 4.980694780264918e-06,
      "loss": 0.0099,
      "reward": 2.3462984561920166,
      "reward_std": 0.5958766937255859,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.4037014842033386,
      "rewards/wrapped_format_reward": 0.75,
      "step": 217
    },
    {
      "completion_length": 500.0,
      "epoch": 43.6,
      "grad_norm": 0.5352597832679749,
      "kl": 0.33760789036750793,
      "learning_rate": 4.980012374199288e-06,
      "loss": 0.0135,
      "reward": 1.1177078485488892,
      "reward_std": 3.422083854675293,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.8822920918464661,
      "rewards/wrapped_format_reward": 0.5,
      "step": 218
    },
    {
      "completion_length": 500.0,
      "epoch": 43.8,
      "grad_norm": 0.78425532579422,
      "kl": 0.45192739367485046,
      "learning_rate": 4.979318164154426e-06,
      "loss": 0.0181,
      "reward": 3.3331549167633057,
      "reward_std": 0.4030221104621887,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.8331548571586609,
      "rewards/wrapped_format_reward": 0.5,
      "step": 219
    },
    {
      "completion_length": 500.0,
      "epoch": 44.0,
      "grad_norm": 0.5511319041252136,
      "kl": 0.2625429630279541,
      "learning_rate": 4.978612153434527e-06,
      "loss": 0.0105,
      "reward": 3.4739222526550293,
      "reward_std": 0.35263335704803467,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.5989223122596741,
      "rewards/wrapped_format_reward": 0.875,
      "step": 220
    },
    {
      "completion_length": 500.0,
      "epoch": 44.2,
      "grad_norm": 0.8345232009887695,
      "kl": 0.5118071436882019,
      "learning_rate": 4.97789434539995e-06,
      "loss": 0.0205,
      "reward": 1.788142442703247,
      "reward_std": 2.3180289268493652,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -1.086857557296753,
      "rewards/wrapped_format_reward": 0.875,
      "step": 221
    },
    {
      "completion_length": 500.0,
      "epoch": 44.4,
      "grad_norm": 0.8292976021766663,
      "kl": 0.5234676003456116,
      "learning_rate": 4.977164743467206e-06,
      "loss": 0.0209,
      "reward": 1.3859682083129883,
      "reward_std": 3.6182608604431152,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.6875,
      "rewards/wrapped_driving_reward": -0.5515317916870117,
      "rewards/wrapped_format_reward": 0.5,
      "step": 222
    },
    {
      "completion_length": 500.0,
      "epoch": 44.6,
      "grad_norm": 0.8200549483299255,
      "kl": 0.3950418531894684,
      "learning_rate": 4.976423351108943e-06,
      "loss": 0.0158,
      "reward": 1.9203238487243652,
      "reward_std": 1.1563453674316406,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.7046762704849243,
      "rewards/wrapped_format_reward": 0.625,
      "step": 223
    },
    {
      "completion_length": 500.0,
      "epoch": 44.8,
      "grad_norm": 0.6968622207641602,
      "kl": 0.2271728217601776,
      "learning_rate": 4.975670171853926e-06,
      "loss": 0.0091,
      "reward": -0.3170052766799927,
      "reward_std": 2.1093220710754395,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9772727489471436,
      "rewards/wrapped_driving_reward": -2.919278144836426,
      "rewards/wrapped_format_reward": 0.625,
      "step": 224
    },
    {
      "completion_length": 500.0,
      "epoch": 45.0,
      "grad_norm": 0.7795050144195557,
      "kl": 0.4355601966381073,
      "learning_rate": 4.97490520928702e-06,
      "loss": 0.0174,
      "reward": 2.6324033737182617,
      "reward_std": 0.5314469933509827,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.11759641766548157,
      "rewards/wrapped_format_reward": 0.75,
      "step": 225
    },
    {
      "completion_length": 500.0,
      "epoch": 45.2,
      "grad_norm": 0.5524005889892578,
      "kl": 0.30952146649360657,
      "learning_rate": 4.974128467049177e-06,
      "loss": 0.0124,
      "reward": -2.125,
      "reward_std": 1.314977765083313,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -4.0,
      "rewards/wrapped_format_reward": 0.375,
      "step": 226
    },
    {
      "completion_length": 500.0,
      "epoch": 45.4,
      "grad_norm": 0.5645884871482849,
      "kl": 0.4939887821674347,
      "learning_rate": 4.9733399488374115e-06,
      "loss": 0.0198,
      "reward": 2.418989658355713,
      "reward_std": 0.14345024526119232,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.08101026713848114,
      "rewards/wrapped_format_reward": 0.5,
      "step": 227
    },
    {
      "completion_length": 500.0,
      "epoch": 45.6,
      "grad_norm": 0.9631263017654419,
      "kl": 0.6647568941116333,
      "learning_rate": 4.972539658404793e-06,
      "loss": 0.0266,
      "reward": -0.0228692889213562,
      "reward_std": 3.135000228881836,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -2.272869110107422,
      "rewards/wrapped_format_reward": 0.75,
      "step": 228
    },
    {
      "completion_length": 500.0,
      "epoch": 45.8,
      "grad_norm": 0.5868902802467346,
      "kl": 0.536701500415802,
      "learning_rate": 4.971727599560418e-06,
      "loss": 0.0215,
      "reward": 2.595135450363159,
      "reward_std": 0.5522119402885437,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.949999988079071,
      "rewards/wrapped_driving_reward": 0.020135482773184776,
      "rewards/wrapped_format_reward": 0.625,
      "step": 229
    },
    {
      "completion_length": 500.0,
      "epoch": 46.0,
      "grad_norm": 0.6927148103713989,
      "kl": 0.5391973257064819,
      "learning_rate": 4.970903776169403e-06,
      "loss": 0.0216,
      "reward": 3.2273426055908203,
      "reward_std": 0.38745206594467163,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9642857313156128,
      "rewards/wrapped_driving_reward": 0.7630569934844971,
      "rewards/wrapped_format_reward": 0.5,
      "step": 230
    },
    {
      "completion_length": 500.0,
      "epoch": 46.2,
      "grad_norm": 2.157358407974243,
      "kl": 0.5963761210441589,
      "learning_rate": 4.9700681921528495e-06,
      "loss": 0.0239,
      "reward": 3.3556950092315674,
      "reward_std": 0.5486971735954285,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.7306950092315674,
      "rewards/wrapped_format_reward": 0.625,
      "step": 231
    },
    {
      "completion_length": 500.0,
      "epoch": 46.4,
      "grad_norm": 0.5409197211265564,
      "kl": 0.31054040789604187,
      "learning_rate": 4.9692208514878445e-06,
      "loss": 0.0124,
      "reward": -1.75,
      "reward_std": 1.1902379989624023,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -4.0,
      "rewards/wrapped_format_reward": 0.75,
      "step": 232
    },
    {
      "completion_length": 500.0,
      "epoch": 46.6,
      "grad_norm": 0.8271388411521912,
      "kl": 0.5030784606933594,
      "learning_rate": 4.968361758207428e-06,
      "loss": 0.0201,
      "reward": 2.2951016426086426,
      "reward_std": 0.6324443817138672,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.07989836484193802,
      "rewards/wrapped_format_reward": 0.375,
      "step": 233
    },
    {
      "completion_length": 500.0,
      "epoch": 46.8,
      "grad_norm": 0.9013113975524902,
      "kl": 0.527148425579071,
      "learning_rate": 4.9674909164005805e-06,
      "loss": 0.0211,
      "reward": -0.08311975002288818,
      "reward_std": 4.243640422821045,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.5831197500228882,
      "rewards/wrapped_format_reward": 0.5,
      "step": 234
    },
    {
      "completion_length": 500.0,
      "epoch": 47.0,
      "grad_norm": 0.621760368347168,
      "kl": 0.5894174575805664,
      "learning_rate": 4.966608330212198e-06,
      "loss": 0.0236,
      "reward": 2.69521427154541,
      "reward_std": 0.2680894732475281,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9722222089767456,
      "rewards/wrapped_driving_reward": -0.15200814604759216,
      "rewards/wrapped_format_reward": 0.875,
      "step": 235
    },
    {
      "completion_length": 500.0,
      "epoch": 47.2,
      "grad_norm": 0.6673128604888916,
      "kl": 0.42412999272346497,
      "learning_rate": 4.965714003843079e-06,
      "loss": 0.017,
      "reward": -2.0,
      "reward_std": 1.0801234245300293,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -4.0,
      "rewards/wrapped_format_reward": 0.5,
      "step": 236
    },
    {
      "completion_length": 500.0,
      "epoch": 47.4,
      "grad_norm": 0.6826753616333008,
      "kl": 0.48437440395355225,
      "learning_rate": 4.9648079415499e-06,
      "loss": 0.0194,
      "reward": 2.6671550273895264,
      "reward_std": 0.6421502828598022,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.20784501731395721,
      "rewards/wrapped_format_reward": 0.875,
      "step": 237
    },
    {
      "completion_length": 500.0,
      "epoch": 47.6,
      "grad_norm": 0.7442097663879395,
      "kl": 0.5179538130760193,
      "learning_rate": 4.963890147645195e-06,
      "loss": 0.0207,
      "reward": 0.023519575595855713,
      "reward_std": 1.7913424968719482,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -2.351480484008789,
      "rewards/wrapped_format_reward": 0.375,
      "step": 238
    },
    {
      "completion_length": 500.0,
      "epoch": 47.8,
      "grad_norm": 0.9971833825111389,
      "kl": 0.2566893994808197,
      "learning_rate": 4.962960626497339e-06,
      "loss": 0.0103,
      "reward": 1.0741076469421387,
      "reward_std": 3.4465811252593994,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.6758923530578613,
      "rewards/wrapped_format_reward": 0.25,
      "step": 239
    },
    {
      "completion_length": 500.0,
      "epoch": 48.0,
      "grad_norm": 0.776371955871582,
      "kl": 0.6667019724845886,
      "learning_rate": 4.962019382530521e-06,
      "loss": 0.0267,
      "reward": 0.7681245803833008,
      "reward_std": 3.63140606880188,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.6068754196166992,
      "rewards/wrapped_format_reward": 0.875,
      "step": 240
    },
    {
      "completion_length": 500.0,
      "epoch": 48.2,
      "grad_norm": 0.958461344242096,
      "kl": 0.6015651226043701,
      "learning_rate": 4.961066420224729e-06,
      "loss": 0.0241,
      "reward": 0.8900174498558044,
      "reward_std": 2.1547889709472656,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9583333134651184,
      "rewards/wrapped_driving_reward": -1.568315863609314,
      "rewards/wrapped_format_reward": 0.5,
      "step": 241
    },
    {
      "completion_length": 500.0,
      "epoch": 48.4,
      "grad_norm": 0.8577614426612854,
      "kl": 0.7052382230758667,
      "learning_rate": 4.960101744115727e-06,
      "loss": 0.0282,
      "reward": 0.500007152557373,
      "reward_std": 3.6019463539123535,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.685606062412262,
      "rewards/wrapped_driving_reward": -1.6855988502502441,
      "rewards/wrapped_format_reward": 0.75,
      "step": 242
    },
    {
      "completion_length": 500.0,
      "epoch": 48.6,
      "grad_norm": 0.6088186502456665,
      "kl": 0.3410260081291199,
      "learning_rate": 4.959125358795031e-06,
      "loss": 0.0136,
      "reward": 1.2359226942062378,
      "reward_std": 3.157292127609253,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.1390773057937622,
      "rewards/wrapped_format_reward": 0.875,
      "step": 243
    },
    {
      "completion_length": 500.0,
      "epoch": 48.8,
      "grad_norm": 0.6780346035957336,
      "kl": 0.47339513897895813,
      "learning_rate": 4.958137268909887e-06,
      "loss": 0.0189,
      "reward": 1.3727295398712158,
      "reward_std": 3.2822999954223633,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.7250000238418579,
      "rewards/wrapped_driving_reward": -0.8522703647613525,
      "rewards/wrapped_format_reward": 0.75,
      "step": 244
    },
    {
      "completion_length": 500.0,
      "epoch": 49.0,
      "grad_norm": 0.6219626069068909,
      "kl": 0.3212871849536896,
      "learning_rate": 4.957137479163253e-06,
      "loss": 0.0129,
      "reward": 0.08353948593139648,
      "reward_std": 2.884551525115967,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -2.2914605140686035,
      "rewards/wrapped_format_reward": 0.875,
      "step": 245
    },
    {
      "completion_length": 500.0,
      "epoch": 49.2,
      "grad_norm": 0.8742188811302185,
      "kl": 0.6009516716003418,
      "learning_rate": 4.956125994313775e-06,
      "loss": 0.024,
      "reward": 3.219036817550659,
      "reward_std": 0.6377858519554138,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.8035714626312256,
      "rewards/wrapped_driving_reward": 0.5404652953147888,
      "rewards/wrapped_format_reward": 0.875,
      "step": 246
    },
    {
      "completion_length": 500.0,
      "epoch": 49.4,
      "grad_norm": 3.8272242546081543,
      "kl": 1.5439887046813965,
      "learning_rate": 4.95510281917576e-06,
      "loss": 0.0618,
      "reward": 3.679497241973877,
      "reward_std": 0.29719072580337524,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.6794970631599426,
      "rewards/wrapped_format_reward": 1.0,
      "step": 247
    },
    {
      "completion_length": 500.0,
      "epoch": 49.6,
      "grad_norm": 0.5449193120002747,
      "kl": 0.3074452579021454,
      "learning_rate": 4.9540679586191605e-06,
      "loss": 0.0123,
      "reward": -0.8099073171615601,
      "reward_std": 2.768624782562256,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -2.9349074363708496,
      "rewards/wrapped_format_reward": 0.625,
      "step": 248
    },
    {
      "completion_length": 500.0,
      "epoch": 49.8,
      "grad_norm": 0.9541939496994019,
      "kl": 0.5022038221359253,
      "learning_rate": 4.953021417569545e-06,
      "loss": 0.0201,
      "reward": 0.9676476120948792,
      "reward_std": 3.3370866775512695,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.1573524475097656,
      "rewards/wrapped_format_reward": 0.625,
      "step": 249
    },
    {
      "completion_length": 500.0,
      "epoch": 50.0,
      "grad_norm": 0.6913716197013855,
      "kl": 0.22377586364746094,
      "learning_rate": 4.9519632010080765e-06,
      "loss": 0.009,
      "reward": -0.7356908917427063,
      "reward_std": 3.197190761566162,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.4856908321380615,
      "rewards/wrapped_format_reward": 0.75,
      "step": 250
    },
    {
      "completion_length": 500.0,
      "epoch": 50.2,
      "grad_norm": 0.8929083347320557,
      "kl": 0.5696563720703125,
      "learning_rate": 4.950893313971492e-06,
      "loss": 0.0228,
      "reward": 1.0284161567687988,
      "reward_std": 2.463923454284668,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -1.7215839624404907,
      "rewards/wrapped_format_reward": 0.75,
      "step": 251
    },
    {
      "completion_length": 500.0,
      "epoch": 50.4,
      "grad_norm": 1.311846375465393,
      "kl": 0.7255131006240845,
      "learning_rate": 4.949811761552074e-06,
      "loss": 0.029,
      "reward": 1.2360342741012573,
      "reward_std": 3.5064666271209717,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.6428571343421936,
      "rewards/wrapped_driving_reward": -0.656822919845581,
      "rewards/wrapped_format_reward": 0.5,
      "step": 252
    },
    {
      "completion_length": 500.0,
      "epoch": 50.6,
      "grad_norm": 0.4827212989330292,
      "kl": 0.33761507272720337,
      "learning_rate": 4.9487185488976284e-06,
      "loss": 0.0135,
      "reward": 1.2470874786376953,
      "reward_std": 3.3136324882507324,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.8779124021530151,
      "rewards/wrapped_format_reward": 0.625,
      "step": 253
    },
    {
      "completion_length": 500.0,
      "epoch": 50.8,
      "grad_norm": 0.846889078617096,
      "kl": 0.7514812350273132,
      "learning_rate": 4.94761368121146e-06,
      "loss": 0.0301,
      "reward": 2.474397897720337,
      "reward_std": 0.5729619264602661,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.96875,
      "rewards/wrapped_driving_reward": -0.24435219168663025,
      "rewards/wrapped_format_reward": 0.75,
      "step": 254
    },
    {
      "completion_length": 467.0,
      "epoch": 51.0,
      "grad_norm": 0.6226330399513245,
      "kl": 0.9035637378692627,
      "learning_rate": 4.9464971637523465e-06,
      "loss": 0.0361,
      "reward": 2.976405620574951,
      "reward_std": 0.597745954990387,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.023594465106725693,
      "rewards/wrapped_format_reward": 1.0,
      "step": 255
    },
    {
      "completion_length": 500.0,
      "epoch": 51.2,
      "grad_norm": 0.6273528933525085,
      "kl": 0.46313872933387756,
      "learning_rate": 4.9453690018345144e-06,
      "loss": 0.0185,
      "reward": 1.4185447692871094,
      "reward_std": 3.323413848876953,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.7064552307128906,
      "rewards/wrapped_format_reward": 0.625,
      "step": 256
    },
    {
      "completion_length": 500.0,
      "epoch": 51.4,
      "grad_norm": 1.4358348846435547,
      "kl": 0.3820998966693878,
      "learning_rate": 4.944229200827616e-06,
      "loss": 0.0153,
      "reward": 0.3355594873428345,
      "reward_std": 2.6716933250427246,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.414440393447876,
      "rewards/wrapped_format_reward": 0.25,
      "step": 257
    },
    {
      "completion_length": 500.0,
      "epoch": 51.6,
      "grad_norm": 0.5715557932853699,
      "kl": 0.26808997988700867,
      "learning_rate": 4.943077766156698e-06,
      "loss": 0.0107,
      "reward": 1.8104004859924316,
      "reward_std": 3.5490846633911133,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.7272727489471436,
      "rewards/wrapped_driving_reward": -0.4168723225593567,
      "rewards/wrapped_format_reward": 0.75,
      "step": 258
    },
    {
      "completion_length": 500.0,
      "epoch": 51.8,
      "grad_norm": 0.9949742555618286,
      "kl": 0.6875662803649902,
      "learning_rate": 4.941914703302181e-06,
      "loss": 0.0275,
      "reward": 3.0740609169006348,
      "reward_std": 1.2247024774551392,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": 0.5740607976913452,
      "rewards/wrapped_format_reward": 0.75,
      "step": 259
    },
    {
      "completion_length": 500.0,
      "epoch": 52.0,
      "grad_norm": 0.5395461916923523,
      "kl": 0.3153356909751892,
      "learning_rate": 4.9407400177998335e-06,
      "loss": 0.0126,
      "reward": 1.252676010131836,
      "reward_std": 3.17907452583313,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.6875,
      "rewards/wrapped_driving_reward": -0.9348239898681641,
      "rewards/wrapped_format_reward": 0.75,
      "step": 260
    },
    {
      "completion_length": 500.0,
      "epoch": 52.2,
      "grad_norm": 0.5175924301147461,
      "kl": 0.41518455743789673,
      "learning_rate": 4.939553715240741e-06,
      "loss": 0.0166,
      "reward": 3.3429102897644043,
      "reward_std": 0.5592035055160522,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.46791017055511475,
      "rewards/wrapped_format_reward": 0.875,
      "step": 261
    },
    {
      "completion_length": 500.0,
      "epoch": 52.4,
      "grad_norm": 0.47460633516311646,
      "kl": 0.5516465306282043,
      "learning_rate": 4.938355801271282e-06,
      "loss": 0.0221,
      "reward": 1.5110602378845215,
      "reward_std": 1.9977182149887085,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9375,
      "rewards/wrapped_driving_reward": -1.3014397621154785,
      "rewards/wrapped_format_reward": 0.875,
      "step": 262
    },
    {
      "completion_length": 500.0,
      "epoch": 52.6,
      "grad_norm": 0.8124420046806335,
      "kl": 0.5848581790924072,
      "learning_rate": 4.937146281593103e-06,
      "loss": 0.0234,
      "reward": 3.247490406036377,
      "reward_std": 0.6556951999664307,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.6224905252456665,
      "rewards/wrapped_format_reward": 0.625,
      "step": 263
    },
    {
      "completion_length": 500.0,
      "epoch": 52.8,
      "grad_norm": 0.6221967339515686,
      "kl": 0.30070972442626953,
      "learning_rate": 4.935925161963089e-06,
      "loss": 0.012,
      "reward": 0.7041885852813721,
      "reward_std": 3.16145658493042,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.295811414718628,
      "rewards/wrapped_format_reward": 0.5,
      "step": 264
    },
    {
      "completion_length": 500.0,
      "epoch": 53.0,
      "grad_norm": 0.8025345206260681,
      "kl": 0.4004298448562622,
      "learning_rate": 4.9346924481933345e-06,
      "loss": 0.016,
      "reward": -2.3282265663146973,
      "reward_std": 1.6561260223388672,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -3.9532265663146973,
      "rewards/wrapped_format_reward": 0.625,
      "step": 265
    },
    {
      "completion_length": 500.0,
      "epoch": 53.2,
      "grad_norm": 2.1500437259674072,
      "kl": 0.9270884990692139,
      "learning_rate": 4.933448146151122e-06,
      "loss": 0.0371,
      "reward": 3.697523355484009,
      "reward_std": 0.2363002598285675,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.8225233554840088,
      "rewards/wrapped_format_reward": 0.875,
      "step": 266
    },
    {
      "completion_length": 500.0,
      "epoch": 53.4,
      "grad_norm": 0.9306778311729431,
      "kl": 0.46557193994522095,
      "learning_rate": 4.932192261758885e-06,
      "loss": 0.0186,
      "reward": 3.163017749786377,
      "reward_std": 0.2954404056072235,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9772727489471436,
      "rewards/wrapped_driving_reward": 0.3107450008392334,
      "rewards/wrapped_format_reward": 0.875,
      "step": 267
    },
    {
      "completion_length": 500.0,
      "epoch": 53.6,
      "grad_norm": 0.7212801575660706,
      "kl": 0.5435701608657837,
      "learning_rate": 4.930924800994192e-06,
      "loss": 0.0217,
      "reward": 1.4288103580474854,
      "reward_std": 3.6827523708343506,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.5711897015571594,
      "rewards/wrapped_format_reward": 0.5,
      "step": 268
    },
    {
      "completion_length": 500.0,
      "epoch": 53.8,
      "grad_norm": 0.8867128491401672,
      "kl": 0.7739095687866211,
      "learning_rate": 4.929645769889704e-06,
      "loss": 0.031,
      "reward": -2.1571238040924072,
      "reward_std": 1.6486133337020874,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -3.7821238040924072,
      "rewards/wrapped_format_reward": 0.625,
      "step": 269
    },
    {
      "completion_length": 500.0,
      "epoch": 54.0,
      "grad_norm": 0.510140061378479,
      "kl": 0.8436269164085388,
      "learning_rate": 4.928355174533153e-06,
      "loss": 0.0337,
      "reward": 2.837273120880127,
      "reward_std": 0.3524271845817566,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.03772694990038872,
      "rewards/wrapped_format_reward": 0.875,
      "step": 270
    },
    {
      "completion_length": 500.0,
      "epoch": 54.2,
      "grad_norm": 0.6447189450263977,
      "kl": 0.37782302498817444,
      "learning_rate": 4.927053021067321e-06,
      "loss": 0.0151,
      "reward": 2.576737642288208,
      "reward_std": 0.5483171939849854,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.326737642288208,
      "rewards/wrapped_format_reward": 0.25,
      "step": 271
    },
    {
      "completion_length": 500.0,
      "epoch": 54.4,
      "grad_norm": 0.670859158039093,
      "kl": 1.0185219049453735,
      "learning_rate": 4.925739315689991e-06,
      "loss": 0.0407,
      "reward": 2.652383327484131,
      "reward_std": 0.27769026160240173,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9166666865348816,
      "rewards/wrapped_driving_reward": -0.1392831951379776,
      "rewards/wrapped_format_reward": 0.875,
      "step": 272
    },
    {
      "completion_length": 500.0,
      "epoch": 54.6,
      "grad_norm": 0.5182738304138184,
      "kl": 0.6035473942756653,
      "learning_rate": 4.924414064653938e-06,
      "loss": 0.0241,
      "reward": -1.0863802433013916,
      "reward_std": 2.398730754852295,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.7272727489471436,
      "rewards/wrapped_driving_reward": -3.188652992248535,
      "rewards/wrapped_format_reward": 0.625,
      "step": 273
    },
    {
      "completion_length": 500.0,
      "epoch": 54.8,
      "grad_norm": 0.506093442440033,
      "kl": 0.23574630916118622,
      "learning_rate": 4.923077274266886e-06,
      "loss": 0.0094,
      "reward": 1.0250887870788574,
      "reward_std": 2.6349871158599854,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -1.8499112129211426,
      "rewards/wrapped_format_reward": 0.875,
      "step": 274
    },
    {
      "completion_length": 500.0,
      "epoch": 55.0,
      "grad_norm": 1.008044719696045,
      "kl": 1.0019625425338745,
      "learning_rate": 4.9217289508914836e-06,
      "loss": 0.0401,
      "reward": 3.194295883178711,
      "reward_std": 0.7734468579292297,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.44429582357406616,
      "rewards/wrapped_format_reward": 0.75,
      "step": 275
    },
    {
      "completion_length": 500.0,
      "epoch": 55.2,
      "grad_norm": 0.7316671013832092,
      "kl": 0.44905000925064087,
      "learning_rate": 4.92036910094527e-06,
      "loss": 0.018,
      "reward": 2.1943540573120117,
      "reward_std": 1.172672152519226,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9750000238418579,
      "rewards/wrapped_driving_reward": -0.40564602613449097,
      "rewards/wrapped_format_reward": 0.625,
      "step": 276
    },
    {
      "completion_length": 500.0,
      "epoch": 55.4,
      "grad_norm": 0.5603216290473938,
      "kl": 0.6424278616905212,
      "learning_rate": 4.91899773090065e-06,
      "loss": 0.0257,
      "reward": 2.5801260471343994,
      "reward_std": 0.4576241075992584,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.16987384855747223,
      "rewards/wrapped_format_reward": 0.75,
      "step": 277
    },
    {
      "completion_length": 500.0,
      "epoch": 55.6,
      "grad_norm": 0.7360312938690186,
      "kl": 0.760132372379303,
      "learning_rate": 4.917614847284858e-06,
      "loss": 0.0304,
      "reward": 3.1857800483703613,
      "reward_std": 0.6251810193061829,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.5607799887657166,
      "rewards/wrapped_format_reward": 0.625,
      "step": 278
    },
    {
      "completion_length": 500.0,
      "epoch": 55.8,
      "grad_norm": 0.6797391176223755,
      "kl": 0.7862927317619324,
      "learning_rate": 4.91622045667993e-06,
      "loss": 0.0315,
      "reward": 1.7073078155517578,
      "reward_std": 3.480921983718872,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.542692244052887,
      "rewards/wrapped_format_reward": 0.75,
      "step": 279
    },
    {
      "completion_length": 500.0,
      "epoch": 56.0,
      "grad_norm": 0.7321208119392395,
      "kl": 0.5703269243240356,
      "learning_rate": 4.914814565722671e-06,
      "loss": 0.0228,
      "reward": 2.8495311737060547,
      "reward_std": 0.2323673665523529,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9583333134651184,
      "rewards/wrapped_driving_reward": 0.01619771495461464,
      "rewards/wrapped_format_reward": 0.875,
      "step": 280
    },
    {
      "completion_length": 500.0,
      "epoch": 56.2,
      "grad_norm": 0.5318320989608765,
      "kl": 0.695686936378479,
      "learning_rate": 4.913397181104623e-06,
      "loss": 0.0278,
      "reward": 1.8786531686782837,
      "reward_std": 3.592921257019043,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.3713468611240387,
      "rewards/wrapped_format_reward": 0.75,
      "step": 281
    },
    {
      "completion_length": 500.0,
      "epoch": 56.4,
      "grad_norm": 1.4247713088989258,
      "kl": 0.6277374029159546,
      "learning_rate": 4.9119683095720325e-06,
      "loss": 0.0251,
      "reward": -1.9102458953857422,
      "reward_std": 3.205610513687134,
      "rewards/mpc_param_extraction_reward": 0.25,
      "rewards/mpc_param_name_reward": 0.25,
      "rewards/wrapped_driving_reward": -3.035245895385742,
      "rewards/wrapped_format_reward": 0.625,
      "step": 282
    },
    {
      "completion_length": 500.0,
      "epoch": 56.6,
      "grad_norm": 0.5404212474822998,
      "kl": 0.5326585173606873,
      "learning_rate": 4.9105279579258234e-06,
      "loss": 0.0213,
      "reward": 2.9429216384887695,
      "reward_std": 0.2231481820344925,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9772727489471436,
      "rewards/wrapped_driving_reward": 0.0906490758061409,
      "rewards/wrapped_format_reward": 0.875,
      "step": 283
    },
    {
      "completion_length": 500.0,
      "epoch": 56.8,
      "grad_norm": 0.649975061416626,
      "kl": 0.692694365978241,
      "learning_rate": 4.909076133021558e-06,
      "loss": 0.0277,
      "reward": 2.443101406097412,
      "reward_std": 0.7186054587364197,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.06810133904218674,
      "rewards/wrapped_format_reward": 0.375,
      "step": 284
    },
    {
      "completion_length": 500.0,
      "epoch": 57.0,
      "grad_norm": 1.1204376220703125,
      "kl": 0.5154175162315369,
      "learning_rate": 4.907612841769407e-06,
      "loss": 0.0206,
      "reward": 3.386246681213379,
      "reward_std": 0.1496068835258484,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.7612467408180237,
      "rewards/wrapped_format_reward": 0.625,
      "step": 285
    },
    {
      "completion_length": 500.0,
      "epoch": 57.2,
      "grad_norm": 0.734048068523407,
      "kl": 0.5711463093757629,
      "learning_rate": 4.906138091134118e-06,
      "loss": 0.0228,
      "reward": 1.2789300680160522,
      "reward_std": 3.544008731842041,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.9710699319839478,
      "rewards/wrapped_format_reward": 0.75,
      "step": 286
    },
    {
      "completion_length": 500.0,
      "epoch": 57.4,
      "grad_norm": 0.5225769281387329,
      "kl": 0.27049189805984497,
      "learning_rate": 4.904651888134982e-06,
      "loss": 0.0108,
      "reward": 3.3136672973632812,
      "reward_std": 0.08986721932888031,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9722222089767456,
      "rewards/wrapped_driving_reward": 0.3414452075958252,
      "rewards/wrapped_format_reward": 1.0,
      "step": 287
    },
    {
      "completion_length": 413.0,
      "epoch": 57.6,
      "grad_norm": 1.317987322807312,
      "kl": 0.8254587650299072,
      "learning_rate": 4.903154239845798e-06,
      "loss": 0.033,
      "reward": 3.501180648803711,
      "reward_std": 0.3513033092021942,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.6261807084083557,
      "rewards/wrapped_format_reward": 0.875,
      "step": 288
    },
    {
      "completion_length": 500.0,
      "epoch": 57.8,
      "grad_norm": 0.6399166584014893,
      "kl": 0.6499161124229431,
      "learning_rate": 4.901645153394838e-06,
      "loss": 0.026,
      "reward": 2.196173667907715,
      "reward_std": 1.2169743776321411,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9545454382896423,
      "rewards/wrapped_driving_reward": -0.38337159156799316,
      "rewards/wrapped_format_reward": 0.625,
      "step": 289
    },
    {
      "completion_length": 465.0,
      "epoch": 58.0,
      "grad_norm": 0.57232266664505,
      "kl": 0.6679652333259583,
      "learning_rate": 4.900124635964823e-06,
      "loss": 0.0267,
      "reward": 2.7008635997772217,
      "reward_std": 0.3644496500492096,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.17413626611232758,
      "rewards/wrapped_format_reward": 0.875,
      "step": 290
    },
    {
      "completion_length": 500.0,
      "epoch": 58.2,
      "grad_norm": 0.5505650639533997,
      "kl": 0.6699390411376953,
      "learning_rate": 4.898592694792871e-06,
      "loss": 0.0268,
      "reward": 3.24006724357605,
      "reward_std": 0.26251503825187683,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.7400672435760498,
      "rewards/wrapped_format_reward": 0.5,
      "step": 291
    },
    {
      "completion_length": 500.0,
      "epoch": 58.4,
      "grad_norm": 0.6135743260383606,
      "kl": 0.41203054785728455,
      "learning_rate": 4.897049337170483e-06,
      "loss": 0.0165,
      "reward": 2.3971381187438965,
      "reward_std": 2.2839035987854004,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.4778619110584259,
      "rewards/wrapped_format_reward": 0.875,
      "step": 292
    },
    {
      "completion_length": 500.0,
      "epoch": 58.6,
      "grad_norm": 0.5831702351570129,
      "kl": 0.4307749271392822,
      "learning_rate": 4.895494570443492e-06,
      "loss": 0.0172,
      "reward": -0.7622057199478149,
      "reward_std": 2.7372794151306152,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -3.0122056007385254,
      "rewards/wrapped_format_reward": 0.75,
      "step": 293
    },
    {
      "completion_length": 500.0,
      "epoch": 58.8,
      "grad_norm": 0.5363391637802124,
      "kl": 0.3049720525741577,
      "learning_rate": 4.8939284020120365e-06,
      "loss": 0.0122,
      "reward": 1.0504395961761475,
      "reward_std": 3.033752202987671,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.699999988079071,
      "rewards/wrapped_driving_reward": -1.274560570716858,
      "rewards/wrapped_format_reward": 0.875,
      "step": 294
    },
    {
      "completion_length": 500.0,
      "epoch": 59.0,
      "grad_norm": 0.6540634632110596,
      "kl": 0.2309640794992447,
      "learning_rate": 4.8923508393305224e-06,
      "loss": 0.0092,
      "reward": 1.0688008069992065,
      "reward_std": 3.047051429748535,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.9311991333961487,
      "rewards/wrapped_format_reward": 0.5,
      "step": 295
    },
    {
      "completion_length": 500.0,
      "epoch": 59.2,
      "grad_norm": 0.7878542542457581,
      "kl": 0.44138067960739136,
      "learning_rate": 4.890761889907589e-06,
      "loss": 0.0177,
      "reward": 1.3176345825195312,
      "reward_std": 2.8866841793060303,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.8073654770851135,
      "rewards/wrapped_format_reward": 0.625,
      "step": 296
    },
    {
      "completion_length": 500.0,
      "epoch": 59.4,
      "grad_norm": 0.5553893446922302,
      "kl": 0.6941342949867249,
      "learning_rate": 4.8891615613060715e-06,
      "loss": 0.0278,
      "reward": 2.6625680923461914,
      "reward_std": 0.46423208713531494,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.037567950785160065,
      "rewards/wrapped_format_reward": 0.625,
      "step": 297
    },
    {
      "completion_length": 500.0,
      "epoch": 59.6,
      "grad_norm": 0.5995137095451355,
      "kl": 0.9473585486412048,
      "learning_rate": 4.887549861142967e-06,
      "loss": 0.0379,
      "reward": 1.7219200134277344,
      "reward_std": 2.298279047012329,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.9030801057815552,
      "rewards/wrapped_format_reward": 0.625,
      "step": 298
    },
    {
      "completion_length": 500.0,
      "epoch": 59.8,
      "grad_norm": 0.58851557970047,
      "kl": 0.5266136527061462,
      "learning_rate": 4.885926797089396e-06,
      "loss": 0.0211,
      "reward": 2.1079962253570557,
      "reward_std": 0.7257985472679138,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.90625,
      "rewards/wrapped_driving_reward": -0.2982538044452667,
      "rewards/wrapped_format_reward": 0.5,
      "step": 299
    },
    {
      "completion_length": 500.0,
      "epoch": 60.0,
      "grad_norm": 0.5463822484016418,
      "kl": 0.4176102578639984,
      "learning_rate": 4.884292376870567e-06,
      "loss": 0.0167,
      "reward": 3.0649795532226562,
      "reward_std": 0.5604602694511414,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.06497950851917267,
      "rewards/wrapped_format_reward": 1.0,
      "step": 300
    },
    {
      "completion_length": 500.0,
      "epoch": 60.2,
      "grad_norm": 0.6283360123634338,
      "kl": 0.7974268198013306,
      "learning_rate": 4.882646608265743e-06,
      "loss": 0.0319,
      "reward": 2.354006290435791,
      "reward_std": 2.2440249919891357,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.6459937691688538,
      "rewards/wrapped_format_reward": 1.0,
      "step": 301
    },
    {
      "completion_length": 500.0,
      "epoch": 60.4,
      "grad_norm": 0.5363072156906128,
      "kl": 0.3808169662952423,
      "learning_rate": 4.880989499108196e-06,
      "loss": 0.0152,
      "reward": 2.4912989139556885,
      "reward_std": 0.2686402499675751,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.2587011158466339,
      "rewards/wrapped_format_reward": 0.75,
      "step": 302
    },
    {
      "completion_length": 500.0,
      "epoch": 60.6,
      "grad_norm": 0.7250503897666931,
      "kl": 0.9471665620803833,
      "learning_rate": 4.8793210572851795e-06,
      "loss": 0.0379,
      "reward": 1.964458703994751,
      "reward_std": 2.0956764221191406,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.6605411767959595,
      "rewards/wrapped_format_reward": 0.625,
      "step": 303
    },
    {
      "completion_length": 500.0,
      "epoch": 60.8,
      "grad_norm": 0.5111416578292847,
      "kl": 0.3088001608848572,
      "learning_rate": 4.8776412907378845e-06,
      "loss": 0.0124,
      "reward": -1.0382410287857056,
      "reward_std": 2.2208359241485596,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -3.038240909576416,
      "rewards/wrapped_format_reward": 0.5,
      "step": 304
    },
    {
      "completion_length": 500.0,
      "epoch": 61.0,
      "grad_norm": 0.5215743780136108,
      "kl": 0.24940745532512665,
      "learning_rate": 4.875950207461403e-06,
      "loss": 0.01,
      "reward": 1.1871674060821533,
      "reward_std": 1.8741310834884644,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -1.1878325939178467,
      "rewards/wrapped_format_reward": 0.375,
      "step": 305
    },
    {
      "completion_length": 500.0,
      "epoch": 61.2,
      "grad_norm": 0.6458624005317688,
      "kl": 0.5968429446220398,
      "learning_rate": 4.874247815504693e-06,
      "loss": 0.0239,
      "reward": 1.3467873334884644,
      "reward_std": 3.565579414367676,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.9032126665115356,
      "rewards/wrapped_format_reward": 0.75,
      "step": 306
    },
    {
      "completion_length": 500.0,
      "epoch": 61.4,
      "grad_norm": 0.5787416696548462,
      "kl": 0.3611050844192505,
      "learning_rate": 4.872534122970536e-06,
      "loss": 0.0144,
      "reward": -1.5641289949417114,
      "reward_std": 2.654411792755127,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.689128875732422,
      "rewards/wrapped_format_reward": 0.125,
      "step": 307
    },
    {
      "completion_length": 500.0,
      "epoch": 61.6,
      "grad_norm": 0.9745285511016846,
      "kl": 1.186964988708496,
      "learning_rate": 4.870809138015499e-06,
      "loss": 0.0475,
      "reward": 3.2458059787750244,
      "reward_std": 0.423534095287323,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.4958060383796692,
      "rewards/wrapped_format_reward": 0.75,
      "step": 308
    },
    {
      "completion_length": 500.0,
      "epoch": 61.8,
      "grad_norm": 0.5002840757369995,
      "kl": 0.35287949442863464,
      "learning_rate": 4.8690728688499e-06,
      "loss": 0.0141,
      "reward": -0.37188810110092163,
      "reward_std": 2.867581367492676,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.699999988079071,
      "rewards/wrapped_driving_reward": -2.821887969970703,
      "rewards/wrapped_format_reward": 1.0,
      "step": 309
    },
    {
      "completion_length": 500.0,
      "epoch": 62.0,
      "grad_norm": 0.5000089406967163,
      "kl": 0.49758777022361755,
      "learning_rate": 4.867325323737765e-06,
      "loss": 0.0199,
      "reward": 1.0681167840957642,
      "reward_std": 3.100673198699951,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.7222222089767456,
      "rewards/wrapped_driving_reward": -1.2791054248809814,
      "rewards/wrapped_format_reward": 0.875,
      "step": 310
    },
    {
      "completion_length": 500.0,
      "epoch": 62.2,
      "grad_norm": 0.5534230470657349,
      "kl": 0.2521771788597107,
      "learning_rate": 4.865566510996787e-06,
      "loss": 0.0101,
      "reward": 1.340180516242981,
      "reward_std": 2.2682108879089355,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -1.2848193645477295,
      "rewards/wrapped_format_reward": 0.625,
      "step": 311
    },
    {
      "completion_length": 500.0,
      "epoch": 62.4,
      "grad_norm": 0.8099197149276733,
      "kl": 0.9075281023979187,
      "learning_rate": 4.863796438998293e-06,
      "loss": 0.0363,
      "reward": 1.2343413829803467,
      "reward_std": 3.5059289932250977,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.7083333134651184,
      "rewards/wrapped_driving_reward": -0.723991870880127,
      "rewards/wrapped_format_reward": 0.5,
      "step": 312
    },
    {
      "completion_length": 500.0,
      "epoch": 62.6,
      "grad_norm": 0.5338905453681946,
      "kl": 0.6918814778327942,
      "learning_rate": 4.862015116167195e-06,
      "loss": 0.0277,
      "reward": -0.8979493975639343,
      "reward_std": 2.7375407218933105,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -3.022949457168579,
      "rewards/wrapped_format_reward": 0.625,
      "step": 313
    },
    {
      "completion_length": 500.0,
      "epoch": 62.8,
      "grad_norm": 0.4916922450065613,
      "kl": 0.46378105878829956,
      "learning_rate": 4.860222550981961e-06,
      "loss": 0.0186,
      "reward": 3.5812926292419434,
      "reward_std": 0.48870983719825745,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.831292450428009,
      "rewards/wrapped_format_reward": 0.75,
      "step": 314
    },
    {
      "completion_length": 500.0,
      "epoch": 63.0,
      "grad_norm": 0.5365267395973206,
      "kl": 0.9468160271644592,
      "learning_rate": 4.858418751974564e-06,
      "loss": 0.0379,
      "reward": 2.7630491256713867,
      "reward_std": 0.2816019356250763,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.013049202039837837,
      "rewards/wrapped_format_reward": 0.75,
      "step": 315
    },
    {
      "completion_length": 500.0,
      "epoch": 63.2,
      "grad_norm": 0.6400690674781799,
      "kl": 0.4425306022167206,
      "learning_rate": 4.856603727730446e-06,
      "loss": 0.0177,
      "reward": 0.8091722130775452,
      "reward_std": 3.2286431789398193,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.7361111044883728,
      "rewards/wrapped_driving_reward": -1.176938772201538,
      "rewards/wrapped_format_reward": 0.5,
      "step": 316
    },
    {
      "completion_length": 500.0,
      "epoch": 63.4,
      "grad_norm": 0.6683816909790039,
      "kl": 0.5423528552055359,
      "learning_rate": 4.854777486888481e-06,
      "loss": 0.0217,
      "reward": 1.423877239227295,
      "reward_std": 2.06416654586792,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9318181872367859,
      "rewards/wrapped_driving_reward": -1.0079410076141357,
      "rewards/wrapped_format_reward": 0.5,
      "step": 317
    },
    {
      "completion_length": 500.0,
      "epoch": 63.6,
      "grad_norm": 0.5277029871940613,
      "kl": 0.49146440625190735,
      "learning_rate": 4.852940038140927e-06,
      "loss": 0.0197,
      "reward": 3.4070889949798584,
      "reward_std": 0.4839623272418976,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.532089114189148,
      "rewards/wrapped_format_reward": 0.875,
      "step": 318
    },
    {
      "completion_length": 500.0,
      "epoch": 63.8,
      "grad_norm": 0.8004822134971619,
      "kl": 0.6069704294204712,
      "learning_rate": 4.8510913902333876e-06,
      "loss": 0.0243,
      "reward": 1.4200356006622314,
      "reward_std": 3.2922091484069824,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.9549642205238342,
      "rewards/wrapped_format_reward": 0.875,
      "step": 319
    },
    {
      "completion_length": 500.0,
      "epoch": 64.0,
      "grad_norm": 0.5641809105873108,
      "kl": 0.47654300928115845,
      "learning_rate": 4.849231551964771e-06,
      "loss": 0.0191,
      "reward": 1.7679204940795898,
      "reward_std": 2.522942543029785,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.48207950592041016,
      "rewards/wrapped_format_reward": 0.25,
      "step": 320
    },
    {
      "completion_length": 500.0,
      "epoch": 64.2,
      "grad_norm": 0.6091551184654236,
      "kl": 0.25179192423820496,
      "learning_rate": 4.8473605321872484e-06,
      "loss": 0.0101,
      "reward": -0.5306634306907654,
      "reward_std": 2.778803825378418,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -2.65566349029541,
      "rewards/wrapped_format_reward": 0.625,
      "step": 321
    },
    {
      "completion_length": 500.0,
      "epoch": 64.4,
      "grad_norm": 0.5598704218864441,
      "kl": 0.5083995461463928,
      "learning_rate": 4.845478339806211e-06,
      "loss": 0.0203,
      "reward": 1.3865933418273926,
      "reward_std": 3.261355400085449,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.6439394354820251,
      "rewards/wrapped_driving_reward": -0.3823460340499878,
      "rewards/wrapped_format_reward": 0.375,
      "step": 322
    },
    {
      "completion_length": 500.0,
      "epoch": 64.6,
      "grad_norm": 1.2819907665252686,
      "kl": 0.26251715421676636,
      "learning_rate": 4.843584983780225e-06,
      "loss": 0.0105,
      "reward": 2.78169846534729,
      "reward_std": 0.5642634034156799,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.53169846534729,
      "rewards/wrapped_format_reward": 0.25,
      "step": 323
    },
    {
      "completion_length": 500.0,
      "epoch": 64.8,
      "grad_norm": 0.5119650363922119,
      "kl": 0.44216257333755493,
      "learning_rate": 4.841680473120994e-06,
      "loss": 0.0177,
      "reward": 1.7560722827911377,
      "reward_std": 0.8860724568367004,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.8999999761581421,
      "rewards/wrapped_driving_reward": -0.893927812576294,
      "rewards/wrapped_format_reward": 0.75,
      "step": 324
    },
    {
      "completion_length": 500.0,
      "epoch": 65.0,
      "grad_norm": 0.5220951437950134,
      "kl": 0.3728632926940918,
      "learning_rate": 4.839764816893315e-06,
      "loss": 0.0149,
      "reward": 0.11694353818893433,
      "reward_std": 3.4642200469970703,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.6625000238418579,
      "rewards/wrapped_driving_reward": -1.7955565452575684,
      "rewards/wrapped_format_reward": 0.5,
      "step": 325
    },
    {
      "completion_length": 500.0,
      "epoch": 65.2,
      "grad_norm": 0.7010544538497925,
      "kl": 0.6249963641166687,
      "learning_rate": 4.83783802421503e-06,
      "loss": 0.025,
      "reward": 2.118408441543579,
      "reward_std": 0.14386098086833954,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.3815915584564209,
      "rewards/wrapped_format_reward": 0.5,
      "step": 326
    },
    {
      "completion_length": 500.0,
      "epoch": 65.4,
      "grad_norm": 0.6010672450065613,
      "kl": 0.8014960289001465,
      "learning_rate": 4.835900104256989e-06,
      "loss": 0.0321,
      "reward": 3.40175724029541,
      "reward_std": 0.4777882695198059,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.7767573595046997,
      "rewards/wrapped_format_reward": 0.625,
      "step": 327
    },
    {
      "completion_length": 500.0,
      "epoch": 65.6,
      "grad_norm": 0.7275899052619934,
      "kl": 1.015032410621643,
      "learning_rate": 4.833951066243004e-06,
      "loss": 0.0406,
      "reward": 1.5494968891143799,
      "reward_std": 3.7214293479919434,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.625,
      "rewards/wrapped_driving_reward": -0.5755031704902649,
      "rewards/wrapped_format_reward": 0.75,
      "step": 328
    },
    {
      "completion_length": 500.0,
      "epoch": 65.8,
      "grad_norm": 0.512077271938324,
      "kl": 0.4679301083087921,
      "learning_rate": 4.831990919449806e-06,
      "loss": 0.0187,
      "reward": 1.843889832496643,
      "reward_std": 1.9196797609329224,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9583333134651184,
      "rewards/wrapped_driving_reward": -0.9894434213638306,
      "rewards/wrapped_format_reward": 0.875,
      "step": 329
    },
    {
      "completion_length": 500.0,
      "epoch": 66.0,
      "grad_norm": 0.5223225355148315,
      "kl": 0.5572952628135681,
      "learning_rate": 4.830019673206997e-06,
      "loss": 0.0223,
      "reward": 1.724566102027893,
      "reward_std": 0.6851178407669067,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.7754338979721069,
      "rewards/wrapped_format_reward": 0.5,
      "step": 330
    },
    {
      "completion_length": 500.0,
      "epoch": 66.2,
      "grad_norm": 0.8261262774467468,
      "kl": 0.6189790964126587,
      "learning_rate": 4.828037336897009e-06,
      "loss": 0.0248,
      "reward": 2.4810099601745605,
      "reward_std": 0.7533643245697021,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9444444179534912,
      "rewards/wrapped_driving_reward": -0.21343453228473663,
      "rewards/wrapped_format_reward": 0.75,
      "step": 331
    },
    {
      "completion_length": 500.0,
      "epoch": 66.4,
      "grad_norm": 0.7530797123908997,
      "kl": 1.0583492517471313,
      "learning_rate": 4.826043919955062e-06,
      "loss": 0.0423,
      "reward": 0.9729395508766174,
      "reward_std": 1.709369421005249,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -1.5270603895187378,
      "rewards/wrapped_format_reward": 0.5,
      "step": 332
    },
    {
      "completion_length": 500.0,
      "epoch": 66.6,
      "grad_norm": 0.7703694105148315,
      "kl": 1.1116454601287842,
      "learning_rate": 4.824039431869112e-06,
      "loss": 0.0445,
      "reward": 2.3814101219177246,
      "reward_std": 0.2855786383152008,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.118589848279953,
      "rewards/wrapped_format_reward": 0.5,
      "step": 333
    },
    {
      "completion_length": 500.0,
      "epoch": 66.8,
      "grad_norm": 1.2055093050003052,
      "kl": 1.0002336502075195,
      "learning_rate": 4.822023882179811e-06,
      "loss": 0.04,
      "reward": 1.3947391510009766,
      "reward_std": 3.263190746307373,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.9802609086036682,
      "rewards/wrapped_format_reward": 0.875,
      "step": 334
    },
    {
      "completion_length": 500.0,
      "epoch": 67.0,
      "grad_norm": 0.5130785703659058,
      "kl": 0.5356588959693909,
      "learning_rate": 4.8199972804804615e-06,
      "loss": 0.0214,
      "reward": 1.5739116668701172,
      "reward_std": 3.7419235706329346,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.7142857313156128,
      "rewards/wrapped_driving_reward": -0.3903741240501404,
      "rewards/wrapped_format_reward": 0.5,
      "step": 335
    },
    {
      "completion_length": 500.0,
      "epoch": 67.2,
      "grad_norm": 0.5589498281478882,
      "kl": 0.5461040139198303,
      "learning_rate": 4.817959636416969e-06,
      "loss": 0.0218,
      "reward": 0.7681459784507751,
      "reward_std": 2.159348726272583,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.949999988079071,
      "rewards/wrapped_driving_reward": -1.931854009628296,
      "rewards/wrapped_format_reward": 0.75,
      "step": 336
    },
    {
      "completion_length": 500.0,
      "epoch": 67.4,
      "grad_norm": 0.5277290344238281,
      "kl": 0.3753708004951477,
      "learning_rate": 4.815910959687795e-06,
      "loss": 0.015,
      "reward": 2.5270836353302,
      "reward_std": 0.9810623526573181,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.22291645407676697,
      "rewards/wrapped_format_reward": 0.75,
      "step": 337
    },
    {
      "completion_length": 500.0,
      "epoch": 67.6,
      "grad_norm": 0.6664614081382751,
      "kl": 0.4376363158226013,
      "learning_rate": 4.8138512600439165e-06,
      "loss": 0.0175,
      "reward": 0.6591283679008484,
      "reward_std": 3.1547491550445557,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.5908715724945068,
      "rewards/wrapped_format_reward": 0.75,
      "step": 338
    },
    {
      "completion_length": 500.0,
      "epoch": 67.8,
      "grad_norm": 0.6150086522102356,
      "kl": 1.0432140827178955,
      "learning_rate": 4.8117805472887706e-06,
      "loss": 0.0417,
      "reward": 1.9501566886901855,
      "reward_std": 3.63374924659729,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.7272727489471436,
      "rewards/wrapped_driving_reward": -0.4021160304546356,
      "rewards/wrapped_format_reward": 0.875,
      "step": 339
    },
    {
      "completion_length": 500.0,
      "epoch": 68.0,
      "grad_norm": 0.6247034072875977,
      "kl": 0.5003357529640198,
      "learning_rate": 4.809698831278217e-06,
      "loss": 0.02,
      "reward": 3.1344668865203857,
      "reward_std": 0.4356966018676758,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.949999988079071,
      "rewards/wrapped_driving_reward": 0.5594670176506042,
      "rewards/wrapped_format_reward": 0.625,
      "step": 340
    },
    {
      "completion_length": 500.0,
      "epoch": 68.2,
      "grad_norm": 0.7365332245826721,
      "kl": 0.9077324867248535,
      "learning_rate": 4.807606121920486e-06,
      "loss": 0.0363,
      "reward": 2.7017669677734375,
      "reward_std": 0.26797592639923096,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.1732332408428192,
      "rewards/wrapped_format_reward": 0.875,
      "step": 341
    },
    {
      "completion_length": 500.0,
      "epoch": 68.4,
      "grad_norm": 0.5844131112098694,
      "kl": 0.7788793444633484,
      "learning_rate": 4.80550242917613e-06,
      "loss": 0.0312,
      "reward": 1.429833173751831,
      "reward_std": 1.8360050916671753,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.930555522441864,
      "rewards/wrapped_driving_reward": -1.2507224082946777,
      "rewards/wrapped_format_reward": 0.75,
      "step": 342
    },
    {
      "completion_length": 500.0,
      "epoch": 68.6,
      "grad_norm": 0.5121277570724487,
      "kl": 0.41953617334365845,
      "learning_rate": 4.803387763057981e-06,
      "loss": 0.0168,
      "reward": 1.4760076999664307,
      "reward_std": 3.3271522521972656,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.7739923000335693,
      "rewards/wrapped_format_reward": 0.75,
      "step": 343
    },
    {
      "completion_length": 500.0,
      "epoch": 68.8,
      "grad_norm": 0.6013743877410889,
      "kl": 0.8165791630744934,
      "learning_rate": 4.801262133631101e-06,
      "loss": 0.0327,
      "reward": 1.8635355234146118,
      "reward_std": 3.9091763496398926,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.3864644765853882,
      "rewards/wrapped_format_reward": 0.75,
      "step": 344
    },
    {
      "completion_length": 500.0,
      "epoch": 69.0,
      "grad_norm": 0.6182828545570374,
      "kl": 1.0937128067016602,
      "learning_rate": 4.799125551012731e-06,
      "loss": 0.0437,
      "reward": 3.419711112976074,
      "reward_std": 0.48218590021133423,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9375,
      "rewards/wrapped_driving_reward": 0.7322112321853638,
      "rewards/wrapped_format_reward": 0.75,
      "step": 345
    },
    {
      "completion_length": 500.0,
      "epoch": 69.2,
      "grad_norm": 0.6208542585372925,
      "kl": 0.6823076605796814,
      "learning_rate": 4.796978025372247e-06,
      "loss": 0.0273,
      "reward": 0.9823777079582214,
      "reward_std": 2.9922034740448,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.7250000238418579,
      "rewards/wrapped_driving_reward": -0.9926222562789917,
      "rewards/wrapped_format_reward": 0.5,
      "step": 346
    },
    {
      "completion_length": 500.0,
      "epoch": 69.4,
      "grad_norm": 0.46940603852272034,
      "kl": 0.6684026122093201,
      "learning_rate": 4.794819566931107e-06,
      "loss": 0.0267,
      "reward": 1.6367030143737793,
      "reward_std": 3.091221809387207,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.4882969856262207,
      "rewards/wrapped_format_reward": 0.625,
      "step": 347
    },
    {
      "completion_length": 500.0,
      "epoch": 69.6,
      "grad_norm": 0.6682089567184448,
      "kl": 0.5266464352607727,
      "learning_rate": 4.79265018596281e-06,
      "loss": 0.0211,
      "reward": 0.8233842253684998,
      "reward_std": 2.3113174438476562,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -1.926615595817566,
      "rewards/wrapped_format_reward": 0.75,
      "step": 348
    },
    {
      "completion_length": 500.0,
      "epoch": 69.8,
      "grad_norm": 0.49833944439888,
      "kl": 0.22715015709400177,
      "learning_rate": 4.79046989279284e-06,
      "loss": 0.0091,
      "reward": 0.9500528573989868,
      "reward_std": 2.9911322593688965,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.1749471426010132,
      "rewards/wrapped_format_reward": 0.625,
      "step": 349
    },
    {
      "completion_length": 500.0,
      "epoch": 70.0,
      "grad_norm": 0.5571979880332947,
      "kl": 0.39211493730545044,
      "learning_rate": 4.788278697798619e-06,
      "loss": 0.0157,
      "reward": 3.3137106895446777,
      "reward_std": 0.5589663982391357,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.8137108087539673,
      "rewards/wrapped_format_reward": 0.5,
      "step": 350
    },
    {
      "completion_length": 500.0,
      "epoch": 70.2,
      "grad_norm": 0.6056340932846069,
      "kl": 0.5785823464393616,
      "learning_rate": 4.7860766114094555e-06,
      "loss": 0.0231,
      "reward": 2.404787063598633,
      "reward_std": 0.34057241678237915,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.22021275758743286,
      "rewards/wrapped_format_reward": 0.625,
      "step": 351
    },
    {
      "completion_length": 500.0,
      "epoch": 70.4,
      "grad_norm": 0.5014784932136536,
      "kl": 0.29856839776039124,
      "learning_rate": 4.783863644106502e-06,
      "loss": 0.0119,
      "reward": 0.22824877500534058,
      "reward_std": 1.6301518678665161,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.8999999761581421,
      "rewards/wrapped_driving_reward": -2.546751022338867,
      "rewards/wrapped_format_reward": 0.875,
      "step": 352
    },
    {
      "completion_length": 500.0,
      "epoch": 70.6,
      "grad_norm": 0.4747620224952698,
      "kl": 0.9486736059188843,
      "learning_rate": 4.781639806422699e-06,
      "loss": 0.0379,
      "reward": 3.8079710006713867,
      "reward_std": 0.04277324676513672,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9791666865348816,
      "rewards/wrapped_driving_reward": 0.8288043737411499,
      "rewards/wrapped_format_reward": 1.0,
      "step": 353
    },
    {
      "completion_length": 500.0,
      "epoch": 70.8,
      "grad_norm": 0.6246031522750854,
      "kl": 0.6103500127792358,
      "learning_rate": 4.779405108942722e-06,
      "loss": 0.0244,
      "reward": 3.2111012935638428,
      "reward_std": 0.5528056621551514,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.46110111474990845,
      "rewards/wrapped_format_reward": 0.75,
      "step": 354
    },
    {
      "completion_length": 500.0,
      "epoch": 71.0,
      "grad_norm": 0.7725012898445129,
      "kl": 1.2677111625671387,
      "learning_rate": 4.77715956230294e-06,
      "loss": 0.0507,
      "reward": -0.5685252547264099,
      "reward_std": 1.8629494905471802,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -3.0685253143310547,
      "rewards/wrapped_format_reward": 0.5,
      "step": 355
    },
    {
      "completion_length": 500.0,
      "epoch": 71.2,
      "grad_norm": 0.8144864439964294,
      "kl": 0.9257476329803467,
      "learning_rate": 4.774903177191358e-06,
      "loss": 0.037,
      "reward": 1.7669270038604736,
      "reward_std": 3.5174825191497803,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.48307299613952637,
      "rewards/wrapped_format_reward": 0.75,
      "step": 356
    },
    {
      "completion_length": 500.0,
      "epoch": 71.4,
      "grad_norm": 0.6923106908798218,
      "kl": 0.52337646484375,
      "learning_rate": 4.77263596434757e-06,
      "loss": 0.0209,
      "reward": -1.5257434844970703,
      "reward_std": 3.4870338439941406,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.4583333134651184,
      "rewards/wrapped_driving_reward": -2.859076738357544,
      "rewards/wrapped_format_reward": 0.375,
      "step": 357
    },
    {
      "completion_length": 500.0,
      "epoch": 71.6,
      "grad_norm": 0.5352271199226379,
      "kl": 0.8994762301445007,
      "learning_rate": 4.770357934562704e-06,
      "loss": 0.036,
      "reward": 2.3911008834838867,
      "reward_std": 0.5525059103965759,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.875,
      "rewards/wrapped_driving_reward": -0.3588991165161133,
      "rewards/wrapped_format_reward": 0.875,
      "step": 358
    },
    {
      "completion_length": 500.0,
      "epoch": 71.8,
      "grad_norm": 0.5663076043128967,
      "kl": 0.7645798325538635,
      "learning_rate": 4.7680690986793734e-06,
      "loss": 0.0306,
      "reward": 1.1860283613204956,
      "reward_std": 3.1243858337402344,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.8139715790748596,
      "rewards/wrapped_format_reward": 0.5,
      "step": 359
    },
    {
      "completion_length": 500.0,
      "epoch": 72.0,
      "grad_norm": 0.45141172409057617,
      "kl": 0.5153231620788574,
      "learning_rate": 4.765769467591626e-06,
      "loss": 0.0206,
      "reward": 2.5078225135803223,
      "reward_std": 0.24923977255821228,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.3671773672103882,
      "rewards/wrapped_format_reward": 0.875,
      "step": 360
    },
    {
      "completion_length": 500.0,
      "epoch": 72.2,
      "grad_norm": 0.4486269950866699,
      "kl": 0.9089427590370178,
      "learning_rate": 4.7634590522448886e-06,
      "loss": 0.0364,
      "reward": 2.8786072731018066,
      "reward_std": 0.16827794909477234,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.121392622590065,
      "rewards/wrapped_format_reward": 1.0,
      "step": 361
    },
    {
      "completion_length": 500.0,
      "epoch": 72.4,
      "grad_norm": 0.5474759340286255,
      "kl": 0.8071030974388123,
      "learning_rate": 4.761137863635921e-06,
      "loss": 0.0323,
      "reward": 2.189277172088623,
      "reward_std": 0.3206322491168976,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.31072288751602173,
      "rewards/wrapped_format_reward": 0.5,
      "step": 362
    },
    {
      "completion_length": 466.0,
      "epoch": 72.6,
      "grad_norm": 1.6158084869384766,
      "kl": 0.6787058711051941,
      "learning_rate": 4.758805912812755e-06,
      "loss": 0.0271,
      "reward": 3.3123486042022705,
      "reward_std": 0.5808764100074768,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.8123486042022705,
      "rewards/wrapped_format_reward": 0.5,
      "step": 363
    },
    {
      "completion_length": 500.0,
      "epoch": 72.8,
      "grad_norm": 0.5949323177337646,
      "kl": 0.9953030943870544,
      "learning_rate": 4.7564632108746524e-06,
      "loss": 0.0398,
      "reward": 1.4721925258636475,
      "reward_std": 2.548034906387329,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.699999988079071,
      "rewards/wrapped_driving_reward": -0.7278074026107788,
      "rewards/wrapped_format_reward": 0.5,
      "step": 364
    },
    {
      "completion_length": 500.0,
      "epoch": 73.0,
      "grad_norm": 0.5318465232849121,
      "kl": 0.7464905977249146,
      "learning_rate": 4.75410976897204e-06,
      "loss": 0.0299,
      "reward": 0.9842851758003235,
      "reward_std": 2.294755697250366,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -1.8907147645950317,
      "rewards/wrapped_format_reward": 0.875,
      "step": 365
    },
    {
      "completion_length": 500.0,
      "epoch": 73.2,
      "grad_norm": 0.6238420009613037,
      "kl": 0.9425249099731445,
      "learning_rate": 4.7517455983064694e-06,
      "loss": 0.0377,
      "reward": 3.152679204940796,
      "reward_std": 0.3899010717868805,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.2776792049407959,
      "rewards/wrapped_format_reward": 0.875,
      "step": 366
    },
    {
      "completion_length": 500.0,
      "epoch": 73.4,
      "grad_norm": 0.5412909984588623,
      "kl": 0.31825780868530273,
      "learning_rate": 4.7493707101305545e-06,
      "loss": 0.0127,
      "reward": -0.11978721618652344,
      "reward_std": 1.6915035247802734,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -2.9947872161865234,
      "rewards/wrapped_format_reward": 0.875,
      "step": 367
    },
    {
      "completion_length": 500.0,
      "epoch": 73.6,
      "grad_norm": 1.3431739807128906,
      "kl": 0.5796495079994202,
      "learning_rate": 4.746985115747918e-06,
      "loss": 0.0232,
      "reward": 2.067376136779785,
      "reward_std": 2.382765769958496,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9861111044883728,
      "rewards/wrapped_driving_reward": -0.6687348484992981,
      "rewards/wrapped_format_reward": 0.75,
      "step": 368
    },
    {
      "completion_length": 467.0,
      "epoch": 73.8,
      "grad_norm": 0.5503766536712646,
      "kl": 0.6919428706169128,
      "learning_rate": 4.744588826513145e-06,
      "loss": 0.0277,
      "reward": 2.56295108795166,
      "reward_std": 0.2011384665966034,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.918749988079071,
      "rewards/wrapped_driving_reward": -0.35579875111579895,
      "rewards/wrapped_format_reward": 1.0,
      "step": 369
    },
    {
      "completion_length": 407.0,
      "epoch": 74.0,
      "grad_norm": 0.5191554427146912,
      "kl": 0.5616594552993774,
      "learning_rate": 4.742181853831721e-06,
      "loss": 0.0225,
      "reward": 3.3623037338256836,
      "reward_std": 0.5844976902008057,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.875,
      "rewards/wrapped_driving_reward": 0.48730385303497314,
      "rewards/wrapped_format_reward": 1.0,
      "step": 370
    },
    {
      "completion_length": 500.0,
      "epoch": 74.2,
      "grad_norm": 0.6369850635528564,
      "kl": 0.351097047328949,
      "learning_rate": 4.739764209159984e-06,
      "loss": 0.014,
      "reward": 0.09611350297927856,
      "reward_std": 1.9310574531555176,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9147727489471436,
      "rewards/wrapped_driving_reward": -2.3186590671539307,
      "rewards/wrapped_format_reward": 0.5,
      "step": 371
    },
    {
      "completion_length": 500.0,
      "epoch": 74.4,
      "grad_norm": 0.592671275138855,
      "kl": 0.7280439734458923,
      "learning_rate": 4.737335904005063e-06,
      "loss": 0.0291,
      "reward": 2.9110307693481445,
      "reward_std": 0.6375144124031067,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.8863636255264282,
      "rewards/wrapped_driving_reward": 0.3996671438217163,
      "rewards/wrapped_format_reward": 0.625,
      "step": 372
    },
    {
      "completion_length": 500.0,
      "epoch": 74.6,
      "grad_norm": 0.5246638059616089,
      "kl": 0.53719562292099,
      "learning_rate": 4.734896949924831e-06,
      "loss": 0.0215,
      "reward": -0.7122367024421692,
      "reward_std": 2.786146402359009,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -2.4622366428375244,
      "rewards/wrapped_format_reward": 0.25,
      "step": 373
    },
    {
      "completion_length": 500.0,
      "epoch": 74.8,
      "grad_norm": 0.47917625308036804,
      "kl": 0.8582153916358948,
      "learning_rate": 4.732447358527843e-06,
      "loss": 0.0343,
      "reward": 0.6898465156555176,
      "reward_std": 2.585700750350952,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9642857313156128,
      "rewards/wrapped_driving_reward": -2.0244390964508057,
      "rewards/wrapped_format_reward": 0.75,
      "step": 374
    },
    {
      "completion_length": 500.0,
      "epoch": 75.0,
      "grad_norm": 2.939560890197754,
      "kl": 1.2595782279968262,
      "learning_rate": 4.729987141473286e-06,
      "loss": 0.0504,
      "reward": 1.0538822412490845,
      "reward_std": 3.369788885116577,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.7083333134651184,
      "rewards/wrapped_driving_reward": -1.1544511318206787,
      "rewards/wrapped_format_reward": 0.75,
      "step": 375
    },
    {
      "completion_length": 500.0,
      "epoch": 75.2,
      "grad_norm": 0.5623316168785095,
      "kl": 0.8516190648078918,
      "learning_rate": 4.72751631047092e-06,
      "loss": 0.0341,
      "reward": 1.0435024499893188,
      "reward_std": 1.7953479290008545,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.8928571343421936,
      "rewards/wrapped_driving_reward": -1.3493547439575195,
      "rewards/wrapped_format_reward": 0.5,
      "step": 376
    },
    {
      "completion_length": 500.0,
      "epoch": 75.4,
      "grad_norm": 0.5572675466537476,
      "kl": 0.5570351481437683,
      "learning_rate": 4.725034877281025e-06,
      "loss": 0.0223,
      "reward": 2.0858306884765625,
      "reward_std": 2.7353217601776123,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.539169430732727,
      "rewards/wrapped_format_reward": 0.625,
      "step": 377
    },
    {
      "completion_length": 500.0,
      "epoch": 75.6,
      "grad_norm": 0.48211756348609924,
      "kl": 0.8747799396514893,
      "learning_rate": 4.7225428537143414e-06,
      "loss": 0.035,
      "reward": 2.519843578338623,
      "reward_std": 0.06863429397344589,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.4801563024520874,
      "rewards/wrapped_format_reward": 1.0,
      "step": 378
    },
    {
      "completion_length": 500.0,
      "epoch": 75.8,
      "grad_norm": 0.5445191860198975,
      "kl": 0.5903018116950989,
      "learning_rate": 4.720040251632019e-06,
      "loss": 0.0236,
      "reward": 2.2911336421966553,
      "reward_std": 0.9245793223381042,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.33386632800102234,
      "rewards/wrapped_format_reward": 0.625,
      "step": 379
    },
    {
      "completion_length": 436.0,
      "epoch": 76.0,
      "grad_norm": 0.494449645280838,
      "kl": 1.1676191091537476,
      "learning_rate": 4.717527082945555e-06,
      "loss": 0.0467,
      "reward": 3.4632315635681152,
      "reward_std": 0.11494097858667374,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9583333134651184,
      "rewards/wrapped_driving_reward": 0.5048981308937073,
      "rewards/wrapped_format_reward": 1.0,
      "step": 380
    },
    {
      "completion_length": 500.0,
      "epoch": 76.2,
      "grad_norm": 0.5274999141693115,
      "kl": 0.5592718124389648,
      "learning_rate": 4.715003359616741e-06,
      "loss": 0.0224,
      "reward": 1.0327037572860718,
      "reward_std": 2.441744804382324,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9642857313156128,
      "rewards/wrapped_driving_reward": -1.931581974029541,
      "rewards/wrapped_format_reward": 1.0,
      "step": 381
    },
    {
      "completion_length": 500.0,
      "epoch": 76.4,
      "grad_norm": 0.4961002469062805,
      "kl": 0.4407399296760559,
      "learning_rate": 4.712469093657605e-06,
      "loss": 0.0176,
      "reward": 0.06372499465942383,
      "reward_std": 4.116382598876953,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -1.6862750053405762,
      "rewards/wrapped_format_reward": 0.75,
      "step": 382
    },
    {
      "completion_length": 500.0,
      "epoch": 76.6,
      "grad_norm": 0.5338679552078247,
      "kl": 0.6065806746482849,
      "learning_rate": 4.709924297130354e-06,
      "loss": 0.0243,
      "reward": 0.48900270462036133,
      "reward_std": 2.7124648094177246,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.6359972953796387,
      "rewards/wrapped_format_reward": 0.625,
      "step": 383
    },
    {
      "completion_length": 500.0,
      "epoch": 76.8,
      "grad_norm": 0.5038735866546631,
      "kl": 0.37462395429611206,
      "learning_rate": 4.707368982147318e-06,
      "loss": 0.015,
      "reward": 1.0094149112701416,
      "reward_std": 3.141808271408081,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.6428571343421936,
      "rewards/wrapped_driving_reward": -0.758442223072052,
      "rewards/wrapped_format_reward": 0.375,
      "step": 384
    },
    {
      "completion_length": 500.0,
      "epoch": 77.0,
      "grad_norm": 0.5533128976821899,
      "kl": 0.5821331143379211,
      "learning_rate": 4.704803160870888e-06,
      "loss": 0.0233,
      "reward": 3.293281078338623,
      "reward_std": 0.5485936999320984,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.6682810187339783,
      "rewards/wrapped_format_reward": 0.625,
      "step": 385
    },
    {
      "completion_length": 500.0,
      "epoch": 77.2,
      "grad_norm": 0.5457403659820557,
      "kl": 0.8627141118049622,
      "learning_rate": 4.702226845513465e-06,
      "loss": 0.0345,
      "reward": 2.2821011543273926,
      "reward_std": 0.8117403984069824,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9090908765792847,
      "rewards/wrapped_driving_reward": -0.12698988616466522,
      "rewards/wrapped_format_reward": 0.5,
      "step": 386
    },
    {
      "completion_length": 500.0,
      "epoch": 77.4,
      "grad_norm": 0.4861312210559845,
      "kl": 0.9649275541305542,
      "learning_rate": 4.699640048337394e-06,
      "loss": 0.0386,
      "reward": 3.2577872276306152,
      "reward_std": 0.1188662126660347,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9375,
      "rewards/wrapped_driving_reward": 0.8202871084213257,
      "rewards/wrapped_format_reward": 0.5,
      "step": 387
    },
    {
      "completion_length": 500.0,
      "epoch": 77.6,
      "grad_norm": 0.5439221858978271,
      "kl": 1.217611312866211,
      "learning_rate": 4.697042781654913e-06,
      "loss": 0.0487,
      "reward": 2.2471086978912354,
      "reward_std": 2.214808225631714,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.6278913021087646,
      "rewards/wrapped_format_reward": 0.875,
      "step": 388
    },
    {
      "completion_length": 500.0,
      "epoch": 77.8,
      "grad_norm": 0.6984148621559143,
      "kl": 0.6679953932762146,
      "learning_rate": 4.694435057828092e-06,
      "loss": 0.0267,
      "reward": 1.991325855255127,
      "reward_std": 0.5920568704605103,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.7586740851402283,
      "rewards/wrapped_format_reward": 0.75,
      "step": 389
    },
    {
      "completion_length": 500.0,
      "epoch": 78.0,
      "grad_norm": 0.5098196268081665,
      "kl": 1.2996803522109985,
      "learning_rate": 4.69181688926877e-06,
      "loss": 0.052,
      "reward": 3.0302999019622803,
      "reward_std": 0.24492628872394562,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.28029996156692505,
      "rewards/wrapped_format_reward": 0.75,
      "step": 390
    },
    {
      "completion_length": 500.0,
      "epoch": 78.2,
      "grad_norm": 0.7863979935646057,
      "kl": 1.2847576141357422,
      "learning_rate": 4.6891882884384994e-06,
      "loss": 0.0514,
      "reward": 1.7419676780700684,
      "reward_std": 3.181126832962036,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.7083333134651184,
      "rewards/wrapped_driving_reward": -0.7163656949996948,
      "rewards/wrapped_format_reward": 1.0,
      "step": 391
    },
    {
      "completion_length": 500.0,
      "epoch": 78.4,
      "grad_norm": 0.8814217448234558,
      "kl": 1.0120067596435547,
      "learning_rate": 4.68654926784849e-06,
      "loss": 0.0405,
      "reward": -1.4409170150756836,
      "reward_std": 3.4238805770874023,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.9409170150756836,
      "rewards/wrapped_format_reward": 0.5,
      "step": 392
    },
    {
      "completion_length": 500.0,
      "epoch": 78.6,
      "grad_norm": 1.0240262746810913,
      "kl": 1.0795583724975586,
      "learning_rate": 4.683899840059543e-06,
      "loss": 0.0432,
      "reward": 1.3624104261398315,
      "reward_std": 2.9232280254364014,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.0125895738601685,
      "rewards/wrapped_format_reward": 0.875,
      "step": 393
    },
    {
      "completion_length": 500.0,
      "epoch": 78.8,
      "grad_norm": 0.5109947919845581,
      "kl": 0.2543693482875824,
      "learning_rate": 4.681240017681994e-06,
      "loss": 0.0102,
      "reward": 1.1824758052825928,
      "reward_std": 3.17807674407959,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.9425241351127625,
      "rewards/wrapped_format_reward": 0.625,
      "step": 394
    },
    {
      "completion_length": 500.0,
      "epoch": 79.0,
      "grad_norm": 0.7424824833869934,
      "kl": 0.7182059288024902,
      "learning_rate": 4.678569813375654e-06,
      "loss": 0.0287,
      "reward": 2.9500770568847656,
      "reward_std": 0.5127381682395935,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9750000238418579,
      "rewards/wrapped_driving_reward": 0.35007697343826294,
      "rewards/wrapped_format_reward": 0.625,
      "step": 395
    },
    {
      "completion_length": 500.0,
      "epoch": 79.2,
      "grad_norm": 0.879162609577179,
      "kl": 0.8275773525238037,
      "learning_rate": 4.675889239849749e-06,
      "loss": 0.0331,
      "reward": 3.0128397941589355,
      "reward_std": 0.4457358717918396,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.3878398835659027,
      "rewards/wrapped_format_reward": 0.625,
      "step": 396
    },
    {
      "completion_length": 500.0,
      "epoch": 79.4,
      "grad_norm": 1.017330527305603,
      "kl": 0.7725871801376343,
      "learning_rate": 4.67319830986286e-06,
      "loss": 0.0309,
      "reward": 1.7406187057495117,
      "reward_std": 3.501706838607788,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.7083333134651184,
      "rewards/wrapped_driving_reward": -0.46771466732025146,
      "rewards/wrapped_format_reward": 0.75,
      "step": 397
    },
    {
      "completion_length": 474.0,
      "epoch": 79.6,
      "grad_norm": 0.6759405732154846,
      "kl": 0.613670289516449,
      "learning_rate": 4.670497036222856e-06,
      "loss": 0.0245,
      "reward": 3.172016143798828,
      "reward_std": 0.8060486316680908,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9166666865348816,
      "rewards/wrapped_driving_reward": 0.3803495764732361,
      "rewards/wrapped_format_reward": 0.875,
      "step": 398
    },
    {
      "completion_length": 500.0,
      "epoch": 79.8,
      "grad_norm": 0.6927025318145752,
      "kl": 0.3530423045158386,
      "learning_rate": 4.667785431786843e-06,
      "loss": 0.0141,
      "reward": 2.0550060272216797,
      "reward_std": 0.7213863134384155,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.8531249761581421,
      "rewards/wrapped_driving_reward": -0.29811891913414,
      "rewards/wrapped_format_reward": 0.5,
      "step": 399
    },
    {
      "completion_length": 500.0,
      "epoch": 80.0,
      "grad_norm": 0.6842610836029053,
      "kl": 0.3291209936141968,
      "learning_rate": 4.665063509461098e-06,
      "loss": 0.0132,
      "reward": -0.5214939117431641,
      "reward_std": 4.023183822631836,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -2.021493911743164,
      "rewards/wrapped_format_reward": 0.5,
      "step": 400
    },
    {
      "completion_length": 500.0,
      "epoch": 80.2,
      "grad_norm": 1.280340552330017,
      "kl": 0.9234899878501892,
      "learning_rate": 4.662331282201002e-06,
      "loss": 0.0369,
      "reward": 1.608371615409851,
      "reward_std": 3.448504686355591,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.6875,
      "rewards/wrapped_driving_reward": -0.7041283845901489,
      "rewards/wrapped_format_reward": 0.875,
      "step": 401
    },
    {
      "completion_length": 500.0,
      "epoch": 80.4,
      "grad_norm": 1.4907759428024292,
      "kl": 0.787459135055542,
      "learning_rate": 4.65958876301099e-06,
      "loss": 0.0315,
      "reward": 3.0962343215942383,
      "reward_std": 0.35777872800827026,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.22123444080352783,
      "rewards/wrapped_format_reward": 0.875,
      "step": 402
    },
    {
      "completion_length": 500.0,
      "epoch": 80.6,
      "grad_norm": 0.49698302149772644,
      "kl": 0.24020951986312866,
      "learning_rate": 4.65683596494448e-06,
      "loss": 0.0096,
      "reward": 2.8298206329345703,
      "reward_std": 0.41156643629074097,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.925000011920929,
      "rewards/wrapped_driving_reward": -0.09517934173345566,
      "rewards/wrapped_format_reward": 1.0,
      "step": 403
    },
    {
      "completion_length": 500.0,
      "epoch": 80.8,
      "grad_norm": 1.796094298362732,
      "kl": 0.980272650718689,
      "learning_rate": 4.654072901103815e-06,
      "loss": 0.0392,
      "reward": 2.3932039737701416,
      "reward_std": 0.9440638422966003,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.356795996427536,
      "rewards/wrapped_format_reward": 0.75,
      "step": 404
    },
    {
      "completion_length": 500.0,
      "epoch": 81.0,
      "grad_norm": 1.1823846101760864,
      "kl": 0.7713034749031067,
      "learning_rate": 4.651299584640198e-06,
      "loss": 0.0309,
      "reward": 2.2802748680114746,
      "reward_std": 0.5997734665870667,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.3447251319885254,
      "rewards/wrapped_format_reward": 0.625,
      "step": 405
    },
    {
      "completion_length": 449.0,
      "epoch": 81.2,
      "grad_norm": 0.6266138553619385,
      "kl": 0.7948797345161438,
      "learning_rate": 4.648516028753632e-06,
      "loss": 0.0318,
      "reward": 2.6703319549560547,
      "reward_std": 0.31966376304626465,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.20466792583465576,
      "rewards/wrapped_format_reward": 0.875,
      "step": 406
    },
    {
      "completion_length": 500.0,
      "epoch": 81.4,
      "grad_norm": 0.7062759399414062,
      "kl": 0.5212233066558838,
      "learning_rate": 4.645722246692856e-06,
      "loss": 0.0208,
      "reward": 1.2558926343917847,
      "reward_std": 3.199486017227173,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.71875,
      "rewards/wrapped_driving_reward": -0.7128572463989258,
      "rewards/wrapped_format_reward": 0.5,
      "step": 407
    },
    {
      "completion_length": 480.0,
      "epoch": 81.6,
      "grad_norm": 0.5641493201255798,
      "kl": 0.8769745826721191,
      "learning_rate": 4.642918251755281e-06,
      "loss": 0.0351,
      "reward": 2.6268346309661865,
      "reward_std": 0.37705758213996887,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.0018346160650253296,
      "rewards/wrapped_format_reward": 0.625,
      "step": 408
    },
    {
      "completion_length": 428.0,
      "epoch": 81.8,
      "grad_norm": 0.6789681315422058,
      "kl": 0.580537736415863,
      "learning_rate": 4.6401040572869295e-06,
      "loss": 0.0232,
      "reward": 3.1963601112365723,
      "reward_std": 0.6234812140464783,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": 0.8213601112365723,
      "rewards/wrapped_format_reward": 0.625,
      "step": 409
    },
    {
      "completion_length": 500.0,
      "epoch": 82.0,
      "grad_norm": 0.5842041373252869,
      "kl": 0.9948064088821411,
      "learning_rate": 4.637279676682367e-06,
      "loss": 0.0398,
      "reward": 2.9659104347229004,
      "reward_std": 0.20523911714553833,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.21591025590896606,
      "rewards/wrapped_format_reward": 0.75,
      "step": 410
    },
    {
      "completion_length": 500.0,
      "epoch": 82.2,
      "grad_norm": 0.7610025405883789,
      "kl": 0.992956280708313,
      "learning_rate": 4.634445123384644e-06,
      "loss": 0.0397,
      "reward": 1.2791435718536377,
      "reward_std": 3.190803050994873,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.9708565473556519,
      "rewards/wrapped_format_reward": 0.75,
      "step": 411
    },
    {
      "completion_length": 500.0,
      "epoch": 82.4,
      "grad_norm": 0.5706319212913513,
      "kl": 1.0067870616912842,
      "learning_rate": 4.631600410885231e-06,
      "loss": 0.0403,
      "reward": 2.5278797149658203,
      "reward_std": 0.3967418074607849,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9772727489471436,
      "rewards/wrapped_driving_reward": -0.19939307868480682,
      "rewards/wrapped_format_reward": 0.75,
      "step": 412
    },
    {
      "completion_length": 500.0,
      "epoch": 82.6,
      "grad_norm": 0.5617349147796631,
      "kl": 0.8165132999420166,
      "learning_rate": 4.6287455527239475e-06,
      "loss": 0.0327,
      "reward": 2.4629299640655518,
      "reward_std": 0.6600992679595947,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9375,
      "rewards/wrapped_driving_reward": 0.27542999386787415,
      "rewards/wrapped_format_reward": 0.25,
      "step": 413
    },
    {
      "completion_length": 500.0,
      "epoch": 82.8,
      "grad_norm": 0.6648067831993103,
      "kl": 0.639345109462738,
      "learning_rate": 4.625880562488908e-06,
      "loss": 0.0256,
      "reward": 3.1534743309020996,
      "reward_std": 0.5813019871711731,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.96875,
      "rewards/wrapped_driving_reward": 0.5597245097160339,
      "rewards/wrapped_format_reward": 0.625,
      "step": 414
    },
    {
      "completion_length": 482.0,
      "epoch": 83.0,
      "grad_norm": 0.5819172263145447,
      "kl": 0.48156633973121643,
      "learning_rate": 4.623005453816447e-06,
      "loss": 0.0193,
      "reward": 0.7172784805297852,
      "reward_std": 2.3905746936798096,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -1.9077215194702148,
      "rewards/wrapped_format_reward": 0.625,
      "step": 415
    },
    {
      "completion_length": 500.0,
      "epoch": 83.2,
      "grad_norm": 0.6890819668769836,
      "kl": 0.970129132270813,
      "learning_rate": 4.620120240391065e-06,
      "loss": 0.0388,
      "reward": 3.3380627632141113,
      "reward_std": 0.025144066661596298,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.7130628824234009,
      "rewards/wrapped_format_reward": 0.625,
      "step": 416
    },
    {
      "completion_length": 500.0,
      "epoch": 83.4,
      "grad_norm": 0.5055440068244934,
      "kl": 1.0465319156646729,
      "learning_rate": 4.617224935945354e-06,
      "loss": 0.0419,
      "reward": 1.7734256982803345,
      "reward_std": 1.9043101072311401,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -1.1015743017196655,
      "rewards/wrapped_format_reward": 0.875,
      "step": 417
    },
    {
      "completion_length": 500.0,
      "epoch": 83.6,
      "grad_norm": 0.7463017106056213,
      "kl": 0.7646268606185913,
      "learning_rate": 4.614319554259934e-06,
      "loss": 0.0306,
      "reward": 0.765255331993103,
      "reward_std": 3.2255241870880127,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.6818181872367859,
      "rewards/wrapped_driving_reward": -1.166562795639038,
      "rewards/wrapped_format_reward": 0.5,
      "step": 418
    },
    {
      "completion_length": 494.0,
      "epoch": 83.8,
      "grad_norm": 0.613161027431488,
      "kl": 1.2434450387954712,
      "learning_rate": 4.611404109163392e-06,
      "loss": 0.0497,
      "reward": 3.17510986328125,
      "reward_std": 0.35877394676208496,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9583333134651184,
      "rewards/wrapped_driving_reward": 0.34177637100219727,
      "rewards/wrapped_format_reward": 0.875,
      "step": 419
    },
    {
      "completion_length": 500.0,
      "epoch": 84.0,
      "grad_norm": 0.4756300151348114,
      "kl": 0.8438820838928223,
      "learning_rate": 4.608478614532215e-06,
      "loss": 0.0338,
      "reward": 2.6704771518707275,
      "reward_std": 1.0551282167434692,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.20452289283275604,
      "rewards/wrapped_format_reward": 0.875,
      "step": 420
    },
    {
      "completion_length": 500.0,
      "epoch": 84.2,
      "grad_norm": 0.6265870332717896,
      "kl": 0.7220280170440674,
      "learning_rate": 4.605543084290716e-06,
      "loss": 0.0289,
      "reward": 3.330387592315674,
      "reward_std": 0.5945422053337097,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.8303877115249634,
      "rewards/wrapped_format_reward": 0.5,
      "step": 421
    },
    {
      "completion_length": 500.0,
      "epoch": 84.4,
      "grad_norm": 0.4972628653049469,
      "kl": 0.4299587309360504,
      "learning_rate": 4.602597532410982e-06,
      "loss": 0.0172,
      "reward": 1.6803350448608398,
      "reward_std": 2.1513352394104004,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.9446649551391602,
      "rewards/wrapped_format_reward": 0.625,
      "step": 422
    },
    {
      "completion_length": 500.0,
      "epoch": 84.6,
      "grad_norm": 1.2557274103164673,
      "kl": 1.1829395294189453,
      "learning_rate": 4.599641972912791e-06,
      "loss": 0.0473,
      "reward": 3.656310796737671,
      "reward_std": 0.22195641696453094,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.7813107967376709,
      "rewards/wrapped_format_reward": 0.875,
      "step": 423
    },
    {
      "completion_length": 500.0,
      "epoch": 84.8,
      "grad_norm": 0.5427276492118835,
      "kl": 0.6175304055213928,
      "learning_rate": 4.596676419863561e-06,
      "loss": 0.0247,
      "reward": 2.078035593032837,
      "reward_std": 1.5830367803573608,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.7969645261764526,
      "rewards/wrapped_format_reward": 0.875,
      "step": 424
    },
    {
      "completion_length": 500.0,
      "epoch": 85.0,
      "grad_norm": 0.6221134066581726,
      "kl": 0.36543771624565125,
      "learning_rate": 4.59370088737827e-06,
      "loss": 0.0146,
      "reward": 3.0459718704223633,
      "reward_std": 0.3525036573410034,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9750000238418579,
      "rewards/wrapped_driving_reward": 0.19597166776657104,
      "rewards/wrapped_format_reward": 0.875,
      "step": 425
    },
    {
      "completion_length": 471.0,
      "epoch": 85.2,
      "grad_norm": 1.1358462572097778,
      "kl": 1.153946042060852,
      "learning_rate": 4.590715389619399e-06,
      "loss": 0.0462,
      "reward": 2.3951056003570557,
      "reward_std": 0.31768667697906494,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.22989457845687866,
      "rewards/wrapped_format_reward": 0.625,
      "step": 426
    },
    {
      "completion_length": 500.0,
      "epoch": 85.4,
      "grad_norm": 0.6305968761444092,
      "kl": 0.5702832341194153,
      "learning_rate": 4.587719940796858e-06,
      "loss": 0.0228,
      "reward": 1.2231757640838623,
      "reward_std": 1.9306583404541016,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -1.5268242359161377,
      "rewards/wrapped_format_reward": 0.75,
      "step": 427
    },
    {
      "completion_length": 500.0,
      "epoch": 85.6,
      "grad_norm": 0.8879334330558777,
      "kl": 0.9042296409606934,
      "learning_rate": 4.584714555167921e-06,
      "loss": 0.0362,
      "reward": 3.215460777282715,
      "reward_std": 0.508735716342926,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9444444179534912,
      "rewards/wrapped_driving_reward": 0.39601635932922363,
      "rewards/wrapped_format_reward": 0.875,
      "step": 428
    },
    {
      "completion_length": 500.0,
      "epoch": 85.8,
      "grad_norm": 0.4813222289085388,
      "kl": 0.408966988325119,
      "learning_rate": 4.581699247037157e-06,
      "loss": 0.0164,
      "reward": 2.7956860065460205,
      "reward_std": 0.7762725353240967,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9285714626312256,
      "rewards/wrapped_driving_reward": 0.11711461842060089,
      "rewards/wrapped_format_reward": 0.75,
      "step": 429
    },
    {
      "completion_length": 500.0,
      "epoch": 86.0,
      "grad_norm": 0.5622531771659851,
      "kl": 0.9757765531539917,
      "learning_rate": 4.578674030756364e-06,
      "loss": 0.039,
      "reward": 3.035037040710449,
      "reward_std": 0.4434798061847687,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.949999988079071,
      "rewards/wrapped_driving_reward": 0.21003687381744385,
      "rewards/wrapped_format_reward": 0.875,
      "step": 430
    },
    {
      "completion_length": 488.0,
      "epoch": 86.2,
      "grad_norm": 0.499776154756546,
      "kl": 1.332625389099121,
      "learning_rate": 4.5756389207244965e-06,
      "loss": 0.0533,
      "reward": 3.2134695053100586,
      "reward_std": 0.827461302280426,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9375,
      "rewards/wrapped_driving_reward": 0.5259695053100586,
      "rewards/wrapped_format_reward": 0.75,
      "step": 431
    },
    {
      "completion_length": 500.0,
      "epoch": 86.4,
      "grad_norm": 0.4789699912071228,
      "kl": 0.47637441754341125,
      "learning_rate": 4.572593931387604e-06,
      "loss": 0.0191,
      "reward": 2.457927703857422,
      "reward_std": 0.9853748083114624,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9375,
      "rewards/wrapped_driving_reward": -0.3545722961425781,
      "rewards/wrapped_format_reward": 0.875,
      "step": 432
    },
    {
      "completion_length": 500.0,
      "epoch": 86.6,
      "grad_norm": 0.4391123354434967,
      "kl": 1.1674070358276367,
      "learning_rate": 4.569539077238756e-06,
      "loss": 0.0467,
      "reward": 2.6357340812683105,
      "reward_std": 0.510990560054779,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.13573402166366577,
      "rewards/wrapped_format_reward": 0.5,
      "step": 433
    },
    {
      "completion_length": 500.0,
      "epoch": 86.8,
      "grad_norm": 0.5584562420845032,
      "kl": 0.7644482851028442,
      "learning_rate": 4.566474372817971e-06,
      "loss": 0.0306,
      "reward": 2.366978645324707,
      "reward_std": 0.4779915511608124,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.13302119076251984,
      "rewards/wrapped_format_reward": 0.5,
      "step": 434
    },
    {
      "completion_length": 500.0,
      "epoch": 87.0,
      "grad_norm": 0.5053340792655945,
      "kl": 0.5127562284469604,
      "learning_rate": 4.5633998327121595e-06,
      "loss": 0.0205,
      "reward": 2.978019952774048,
      "reward_std": 0.3388763666152954,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.02198006771504879,
      "rewards/wrapped_format_reward": 1.0,
      "step": 435
    },
    {
      "completion_length": 500.0,
      "epoch": 87.2,
      "grad_norm": 0.5881168246269226,
      "kl": 0.8654472231864929,
      "learning_rate": 4.560315471555039e-06,
      "loss": 0.0346,
      "reward": 2.695511817932129,
      "reward_std": 0.2650168240070343,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.17948828637599945,
      "rewards/wrapped_format_reward": 0.875,
      "step": 436
    },
    {
      "completion_length": 500.0,
      "epoch": 87.4,
      "grad_norm": 0.5051465630531311,
      "kl": 0.7103445529937744,
      "learning_rate": 4.557221304027077e-06,
      "loss": 0.0284,
      "reward": 0.450472354888916,
      "reward_std": 2.2633330821990967,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -1.924527645111084,
      "rewards/wrapped_format_reward": 0.375,
      "step": 437
    },
    {
      "completion_length": 500.0,
      "epoch": 87.6,
      "grad_norm": 0.8089684247970581,
      "kl": 0.7311402559280396,
      "learning_rate": 4.55411734485541e-06,
      "loss": 0.0292,
      "reward": 3.2812561988830566,
      "reward_std": 0.32883548736572266,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.4062563478946686,
      "rewards/wrapped_format_reward": 0.875,
      "step": 438
    },
    {
      "completion_length": 500.0,
      "epoch": 87.8,
      "grad_norm": 0.5592673420906067,
      "kl": 0.6468991041183472,
      "learning_rate": 4.551003608813784e-06,
      "loss": 0.0259,
      "reward": 1.8620704412460327,
      "reward_std": 0.6889193058013916,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9750000238418579,
      "rewards/wrapped_driving_reward": -0.8629295229911804,
      "rewards/wrapped_format_reward": 0.75,
      "step": 439
    },
    {
      "completion_length": 500.0,
      "epoch": 88.0,
      "grad_norm": 0.5054028630256653,
      "kl": 0.36086270213127136,
      "learning_rate": 4.54788011072248e-06,
      "loss": 0.0144,
      "reward": 3.3388941287994385,
      "reward_std": 0.5939581394195557,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.8388940691947937,
      "rewards/wrapped_format_reward": 0.5,
      "step": 440
    },
    {
      "completion_length": 500.0,
      "epoch": 88.2,
      "grad_norm": 9.32406997680664,
      "kl": 2.2713303565979004,
      "learning_rate": 4.544746865448239e-06,
      "loss": 0.0909,
      "reward": 1.8941614627838135,
      "reward_std": 2.3242433071136475,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.7308385968208313,
      "rewards/wrapped_format_reward": 0.625,
      "step": 441
    },
    {
      "completion_length": 500.0,
      "epoch": 88.4,
      "grad_norm": 0.7286486625671387,
      "kl": 0.23764638602733612,
      "learning_rate": 4.541603887904198e-06,
      "loss": 0.0095,
      "reward": 1.595595121383667,
      "reward_std": 3.398191213607788,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.65625,
      "rewards/wrapped_driving_reward": -0.43565481901168823,
      "rewards/wrapped_format_reward": 0.625,
      "step": 442
    },
    {
      "completion_length": 472.0,
      "epoch": 88.6,
      "grad_norm": 0.4791865944862366,
      "kl": 0.853591799736023,
      "learning_rate": 4.538451193049814e-06,
      "loss": 0.0341,
      "reward": 2.5504026412963867,
      "reward_std": 0.576043963432312,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.4495972692966461,
      "rewards/wrapped_format_reward": 1.0,
      "step": 443
    },
    {
      "completion_length": 500.0,
      "epoch": 88.8,
      "grad_norm": 0.5535397529602051,
      "kl": 0.1802796721458435,
      "learning_rate": 4.535288795890799e-06,
      "loss": 0.0072,
      "reward": -0.1438617706298828,
      "reward_std": 3.8885178565979004,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.44999998807907104,
      "rewards/wrapped_driving_reward": -1.593861699104309,
      "rewards/wrapped_format_reward": 0.5,
      "step": 444
    },
    {
      "completion_length": 500.0,
      "epoch": 89.0,
      "grad_norm": 0.5480430126190186,
      "kl": 0.5194934010505676,
      "learning_rate": 4.532116711479039e-06,
      "loss": 0.0208,
      "reward": 2.7844161987304688,
      "reward_std": 0.34969019889831543,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.0905836746096611,
      "rewards/wrapped_format_reward": 0.875,
      "step": 445
    },
    {
      "completion_length": 500.0,
      "epoch": 89.2,
      "grad_norm": 7.007938861846924,
      "kl": 1.402961015701294,
      "learning_rate": 4.528934954912531e-06,
      "loss": 0.0561,
      "reward": -0.7073632478713989,
      "reward_std": 2.505068778991699,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.7083333134651184,
      "rewards/wrapped_driving_reward": -3.040696620941162,
      "rewards/wrapped_format_reward": 0.875,
      "step": 446
    },
    {
      "completion_length": 500.0,
      "epoch": 89.4,
      "grad_norm": 0.558776319026947,
      "kl": 1.3707541227340698,
      "learning_rate": 4.525743541335309e-06,
      "loss": 0.0548,
      "reward": 2.7044730186462402,
      "reward_std": 0.5199458003044128,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.2955269515514374,
      "rewards/wrapped_format_reward": 1.0,
      "step": 447
    },
    {
      "completion_length": 500.0,
      "epoch": 89.6,
      "grad_norm": 0.46237713098526,
      "kl": 0.986226499080658,
      "learning_rate": 4.522542485937369e-06,
      "loss": 0.0394,
      "reward": 3.5677292346954346,
      "reward_std": 0.30159273743629456,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.817729115486145,
      "rewards/wrapped_format_reward": 0.75,
      "step": 448
    },
    {
      "completion_length": 500.0,
      "epoch": 89.8,
      "grad_norm": 0.6039856672286987,
      "kl": 0.5541295409202576,
      "learning_rate": 4.519331803954599e-06,
      "loss": 0.0222,
      "reward": 2.4867310523986816,
      "reward_std": 0.18556839227676392,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9147727489471436,
      "rewards/wrapped_driving_reward": -0.17804156243801117,
      "rewards/wrapped_format_reward": 0.75,
      "step": 449
    },
    {
      "completion_length": 500.0,
      "epoch": 90.0,
      "grad_norm": 0.6416642665863037,
      "kl": 0.5888490676879883,
      "learning_rate": 4.516111510668707e-06,
      "loss": 0.0236,
      "reward": 3.0717105865478516,
      "reward_std": 0.9419719576835632,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.5717105865478516,
      "rewards/wrapped_format_reward": 0.5,
      "step": 450
    },
    {
      "completion_length": 500.0,
      "epoch": 90.2,
      "grad_norm": 0.8580331802368164,
      "kl": 1.3547351360321045,
      "learning_rate": 4.512881621407146e-06,
      "loss": 0.0542,
      "reward": 1.4438624382019043,
      "reward_std": 3.3256571292877197,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.9311375021934509,
      "rewards/wrapped_format_reward": 0.875,
      "step": 451
    },
    {
      "completion_length": 500.0,
      "epoch": 90.4,
      "grad_norm": 0.6190384030342102,
      "kl": 0.4166165888309479,
      "learning_rate": 4.509642151543043e-06,
      "loss": 0.0167,
      "reward": 1.4482831954956055,
      "reward_std": 3.3409366607666016,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.9267167448997498,
      "rewards/wrapped_format_reward": 0.875,
      "step": 452
    },
    {
      "completion_length": 470.0,
      "epoch": 90.6,
      "grad_norm": 0.9551142454147339,
      "kl": 0.9795337915420532,
      "learning_rate": 4.506393116495128e-06,
      "loss": 0.0392,
      "reward": 3.8107566833496094,
      "reward_std": 0.042369965463876724,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9750000238418579,
      "rewards/wrapped_driving_reward": 0.835756778717041,
      "rewards/wrapped_format_reward": 1.0,
      "step": 453
    },
    {
      "completion_length": 500.0,
      "epoch": 90.8,
      "grad_norm": 0.532974362373352,
      "kl": 0.43851980566978455,
      "learning_rate": 4.503134531727652e-06,
      "loss": 0.0175,
      "reward": 2.5775976181030273,
      "reward_std": 0.30674323439598083,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.17240235209465027,
      "rewards/wrapped_format_reward": 0.75,
      "step": 454
    },
    {
      "completion_length": 500.0,
      "epoch": 91.0,
      "grad_norm": 0.5811918377876282,
      "kl": 0.46373996138572693,
      "learning_rate": 4.499866412750324e-06,
      "loss": 0.0185,
      "reward": 3.5562117099761963,
      "reward_std": 0.12775082886219025,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.5562118291854858,
      "rewards/wrapped_format_reward": 1.0,
      "step": 455
    },
    {
      "completion_length": 500.0,
      "epoch": 91.2,
      "grad_norm": 0.53252774477005,
      "kl": 0.653654158115387,
      "learning_rate": 4.496588775118232e-06,
      "loss": 0.0261,
      "reward": 1.345137119293213,
      "reward_std": 1.6006327867507935,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9750000238418579,
      "rewards/wrapped_driving_reward": -1.2548627853393555,
      "rewards/wrapped_format_reward": 0.625,
      "step": 456
    },
    {
      "completion_length": 500.0,
      "epoch": 91.4,
      "grad_norm": 0.6192057728767395,
      "kl": 0.8161628842353821,
      "learning_rate": 4.493301634431768e-06,
      "loss": 0.0326,
      "reward": 3.238480567932129,
      "reward_std": 0.6025396585464478,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9750000238418579,
      "rewards/wrapped_driving_reward": 0.513480544090271,
      "rewards/wrapped_format_reward": 0.75,
      "step": 457
    },
    {
      "completion_length": 500.0,
      "epoch": 91.6,
      "grad_norm": 0.5338960289955139,
      "kl": 0.4019594192504883,
      "learning_rate": 4.490005006336555e-06,
      "loss": 0.0161,
      "reward": 1.511120080947876,
      "reward_std": 3.7101433277130127,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.6138800382614136,
      "rewards/wrapped_format_reward": 0.625,
      "step": 458
    },
    {
      "completion_length": 500.0,
      "epoch": 91.8,
      "grad_norm": 0.6177345514297485,
      "kl": 0.7023136615753174,
      "learning_rate": 4.486698906523375e-06,
      "loss": 0.0281,
      "reward": 0.6668235063552856,
      "reward_std": 2.7802770137786865,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.7022727131843567,
      "rewards/wrapped_driving_reward": -1.1604492664337158,
      "rewards/wrapped_format_reward": 0.375,
      "step": 459
    },
    {
      "completion_length": 500.0,
      "epoch": 92.0,
      "grad_norm": 0.498141884803772,
      "kl": 0.7440797686576843,
      "learning_rate": 4.4833833507280884e-06,
      "loss": 0.0298,
      "reward": 1.770745038986206,
      "reward_std": 2.273341178894043,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.854254961013794,
      "rewards/wrapped_format_reward": 0.625,
      "step": 460
    },
    {
      "completion_length": 500.0,
      "epoch": 92.2,
      "grad_norm": 0.5320531725883484,
      "kl": 0.5814719796180725,
      "learning_rate": 4.4800583547315654e-06,
      "loss": 0.0233,
      "reward": -0.1779249906539917,
      "reward_std": 1.991480827331543,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -3.0529251098632812,
      "rewards/wrapped_format_reward": 0.875,
      "step": 461
    },
    {
      "completion_length": 500.0,
      "epoch": 92.4,
      "grad_norm": 0.6644296646118164,
      "kl": 1.252918004989624,
      "learning_rate": 4.476723934359609e-06,
      "loss": 0.0501,
      "reward": 1.581032395362854,
      "reward_std": 3.7278060913085938,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.6689676642417908,
      "rewards/wrapped_format_reward": 0.75,
      "step": 462
    },
    {
      "completion_length": 500.0,
      "epoch": 92.6,
      "grad_norm": 0.6294512748718262,
      "kl": 0.454416960477829,
      "learning_rate": 4.473380105482875e-06,
      "loss": 0.0182,
      "reward": 0.13677752017974854,
      "reward_std": 2.944185733795166,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -2.113222360610962,
      "rewards/wrapped_format_reward": 0.75,
      "step": 463
    },
    {
      "completion_length": 500.0,
      "epoch": 92.8,
      "grad_norm": 0.5402928590774536,
      "kl": 0.8170625567436218,
      "learning_rate": 4.470026884016805e-06,
      "loss": 0.0327,
      "reward": 3.565312385559082,
      "reward_std": 0.3042621612548828,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.8153125047683716,
      "rewards/wrapped_format_reward": 0.75,
      "step": 464
    },
    {
      "completion_length": 500.0,
      "epoch": 93.0,
      "grad_norm": 0.533676266670227,
      "kl": 0.7118498682975769,
      "learning_rate": 4.466664285921543e-06,
      "loss": 0.0285,
      "reward": 2.770576000213623,
      "reward_std": 0.16069670021533966,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9545454978942871,
      "rewards/wrapped_driving_reward": -0.05896963179111481,
      "rewards/wrapped_format_reward": 0.875,
      "step": 465
    },
    {
      "completion_length": 500.0,
      "epoch": 93.2,
      "grad_norm": 0.5372627377510071,
      "kl": 1.061421275138855,
      "learning_rate": 4.463292327201862e-06,
      "loss": 0.0425,
      "reward": 0.8557369112968445,
      "reward_std": 2.914240598678589,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.3942632675170898,
      "rewards/wrapped_format_reward": 0.75,
      "step": 466
    },
    {
      "completion_length": 500.0,
      "epoch": 93.4,
      "grad_norm": 0.6193510293960571,
      "kl": 1.1515281200408936,
      "learning_rate": 4.459911023907092e-06,
      "loss": 0.0461,
      "reward": 3.183587074279785,
      "reward_std": 0.39359328150749207,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9472222328186035,
      "rewards/wrapped_driving_reward": 0.4863646626472473,
      "rewards/wrapped_format_reward": 0.75,
      "step": 467
    },
    {
      "completion_length": 500.0,
      "epoch": 93.6,
      "grad_norm": 0.5153403878211975,
      "kl": 0.6276007890701294,
      "learning_rate": 4.456520392131035e-06,
      "loss": 0.0251,
      "reward": 1.2580312490463257,
      "reward_std": 2.2587177753448486,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -1.6169687509536743,
      "rewards/wrapped_format_reward": 0.875,
      "step": 468
    },
    {
      "completion_length": 500.0,
      "epoch": 93.8,
      "grad_norm": 0.8933354616165161,
      "kl": 0.5966981053352356,
      "learning_rate": 4.453120448011897e-06,
      "loss": 0.0239,
      "reward": 3.828160285949707,
      "reward_std": 0.019492290914058685,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.828160285949707,
      "rewards/wrapped_format_reward": 1.0,
      "step": 469
    },
    {
      "completion_length": 500.0,
      "epoch": 94.0,
      "grad_norm": 0.5882790684700012,
      "kl": 0.6073835492134094,
      "learning_rate": 4.4497112077322045e-06,
      "loss": 0.0243,
      "reward": 2.9608585834503174,
      "reward_std": 0.5855686664581299,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9791666865348816,
      "rewards/wrapped_driving_reward": 0.2316918969154358,
      "rewards/wrapped_format_reward": 0.75,
      "step": 470
    },
    {
      "completion_length": 500.0,
      "epoch": 94.2,
      "grad_norm": 0.5647352337837219,
      "kl": 0.695755660533905,
      "learning_rate": 4.446292687518734e-06,
      "loss": 0.0278,
      "reward": 1.5310817956924438,
      "reward_std": 3.7384822368621826,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.6785714626312256,
      "rewards/wrapped_driving_reward": -0.39748966693878174,
      "rewards/wrapped_format_reward": 0.5,
      "step": 471
    },
    {
      "completion_length": 500.0,
      "epoch": 94.4,
      "grad_norm": 0.5346417427062988,
      "kl": 0.8996866345405579,
      "learning_rate": 4.442864903642428e-06,
      "loss": 0.036,
      "reward": 1.0604596138000488,
      "reward_std": 3.060889959335327,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.625,
      "rewards/wrapped_driving_reward": -1.0645403861999512,
      "rewards/wrapped_format_reward": 0.75,
      "step": 472
    },
    {
      "completion_length": 500.0,
      "epoch": 94.6,
      "grad_norm": 0.5798156261444092,
      "kl": 0.6437966227531433,
      "learning_rate": 4.439427872418321e-06,
      "loss": 0.0258,
      "reward": 3.384183168411255,
      "reward_std": 0.42769670486450195,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.5091832280158997,
      "rewards/wrapped_format_reward": 0.875,
      "step": 473
    },
    {
      "completion_length": 500.0,
      "epoch": 94.8,
      "grad_norm": 0.49952396750450134,
      "kl": 0.5218140482902527,
      "learning_rate": 4.435981610205464e-06,
      "loss": 0.0209,
      "reward": -0.285653293132782,
      "reward_std": 2.212059497833252,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9583333134651184,
      "rewards/wrapped_driving_reward": -2.9939866065979004,
      "rewards/wrapped_format_reward": 0.75,
      "step": 474
    },
    {
      "completion_length": 500.0,
      "epoch": 95.0,
      "grad_norm": 1.0312459468841553,
      "kl": 0.6847164630889893,
      "learning_rate": 4.432526133406843e-06,
      "loss": 0.0274,
      "reward": 2.879366636276245,
      "reward_std": 0.39730778336524963,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.2543666362762451,
      "rewards/wrapped_format_reward": 0.625,
      "step": 475
    },
    {
      "completion_length": 500.0,
      "epoch": 95.2,
      "grad_norm": 0.582595705986023,
      "kl": 0.599985659122467,
      "learning_rate": 4.4290614584693005e-06,
      "loss": 0.024,
      "reward": 0.7622314691543579,
      "reward_std": 2.525625228881836,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -1.237768530845642,
      "rewards/wrapped_format_reward": 0.5,
      "step": 476
    },
    {
      "completion_length": 500.0,
      "epoch": 95.4,
      "grad_norm": 0.5313723683357239,
      "kl": 0.868817925453186,
      "learning_rate": 4.425587601883461e-06,
      "loss": 0.0348,
      "reward": 2.211709976196289,
      "reward_std": 0.4423002600669861,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.96875,
      "rewards/wrapped_driving_reward": -0.3820401132106781,
      "rewards/wrapped_format_reward": 0.625,
      "step": 477
    },
    {
      "completion_length": 500.0,
      "epoch": 95.6,
      "grad_norm": 0.5549596548080444,
      "kl": 1.2767629623413086,
      "learning_rate": 4.422104580183649e-06,
      "loss": 0.0511,
      "reward": 3.195434331893921,
      "reward_std": 0.395979642868042,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.875,
      "rewards/wrapped_driving_reward": 0.4454343914985657,
      "rewards/wrapped_format_reward": 0.875,
      "step": 478
    },
    {
      "completion_length": 500.0,
      "epoch": 95.8,
      "grad_norm": 0.4778159260749817,
      "kl": 0.23531889915466309,
      "learning_rate": 4.418612409947814e-06,
      "loss": 0.0094,
      "reward": 2.747316360473633,
      "reward_std": 0.6012102961540222,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.122316375374794,
      "rewards/wrapped_format_reward": 0.625,
      "step": 479
    },
    {
      "completion_length": 500.0,
      "epoch": 96.0,
      "grad_norm": 0.6010138392448425,
      "kl": 0.6477577090263367,
      "learning_rate": 4.415111107797445e-06,
      "loss": 0.0259,
      "reward": 1.7465873956680298,
      "reward_std": 3.5261001586914062,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.3784126043319702,
      "rewards/wrapped_format_reward": 0.625,
      "step": 480
    },
    {
      "completion_length": 476.0,
      "epoch": 96.2,
      "grad_norm": 0.4925939440727234,
      "kl": 1.1028554439544678,
      "learning_rate": 4.4116006903975015e-06,
      "loss": 0.0441,
      "reward": 2.4060165882110596,
      "reward_std": 0.4904329478740692,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.984375,
      "rewards/wrapped_driving_reward": -0.20335841178894043,
      "rewards/wrapped_format_reward": 0.625,
      "step": 481
    },
    {
      "completion_length": 467.0,
      "epoch": 96.4,
      "grad_norm": 0.5640705823898315,
      "kl": 1.066386103630066,
      "learning_rate": 4.408081174456322e-06,
      "loss": 0.0427,
      "reward": 3.762505292892456,
      "reward_std": 0.12346359342336655,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9375,
      "rewards/wrapped_driving_reward": 0.825005292892456,
      "rewards/wrapped_format_reward": 1.0,
      "step": 482
    },
    {
      "completion_length": 500.0,
      "epoch": 96.6,
      "grad_norm": 0.5508888959884644,
      "kl": 0.3224889039993286,
      "learning_rate": 4.404552576725557e-06,
      "loss": 0.0129,
      "reward": 1.1607617139816284,
      "reward_std": 3.1784019470214844,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.7142382860183716,
      "rewards/wrapped_format_reward": 0.375,
      "step": 483
    },
    {
      "completion_length": 500.0,
      "epoch": 96.8,
      "grad_norm": 0.5421119332313538,
      "kl": 0.2217361032962799,
      "learning_rate": 4.401014914000078e-06,
      "loss": 0.0089,
      "reward": 1.3363996744155884,
      "reward_std": 3.2391209602355957,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.7886003255844116,
      "rewards/wrapped_format_reward": 0.625,
      "step": 484
    },
    {
      "completion_length": 403.0,
      "epoch": 97.0,
      "grad_norm": 0.5398365259170532,
      "kl": 1.3086752891540527,
      "learning_rate": 4.397468203117905e-06,
      "loss": 0.0523,
      "reward": 2.178347110748291,
      "reward_std": 2.1330792903900146,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.571652889251709,
      "rewards/wrapped_format_reward": 0.75,
      "step": 485
    },
    {
      "completion_length": 500.0,
      "epoch": 97.2,
      "grad_norm": 0.5164806246757507,
      "kl": 1.1992006301879883,
      "learning_rate": 4.393912460960125e-06,
      "loss": 0.048,
      "reward": -0.017039000988006592,
      "reward_std": 1.9406579732894897,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.869949460029602,
      "rewards/wrapped_driving_reward": -2.386988401412964,
      "rewards/wrapped_format_reward": 0.5,
      "step": 486
    },
    {
      "completion_length": 447.0,
      "epoch": 97.4,
      "grad_norm": 0.5225284695625305,
      "kl": 0.9045883417129517,
      "learning_rate": 4.3903477044508066e-06,
      "loss": 0.0362,
      "reward": 3.6854019165039062,
      "reward_std": 0.25948864221572876,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.8104017972946167,
      "rewards/wrapped_format_reward": 0.875,
      "step": 487
    },
    {
      "completion_length": 500.0,
      "epoch": 97.6,
      "grad_norm": 0.5833526253700256,
      "kl": 0.5774132013320923,
      "learning_rate": 4.386773950556931e-06,
      "loss": 0.0231,
      "reward": 3.013421058654785,
      "reward_std": 0.4619700014591217,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9583333134651184,
      "rewards/wrapped_driving_reward": 0.18008768558502197,
      "rewards/wrapped_format_reward": 0.875,
      "step": 488
    },
    {
      "completion_length": 500.0,
      "epoch": 97.8,
      "grad_norm": 0.4881477653980255,
      "kl": 0.38699668645858765,
      "learning_rate": 4.3831912162882946e-06,
      "loss": 0.0155,
      "reward": 0.5115635395050049,
      "reward_std": 1.8448858261108398,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.8522727489471436,
      "rewards/wrapped_driving_reward": -1.9657092094421387,
      "rewards/wrapped_format_reward": 0.625,
      "step": 489
    },
    {
      "completion_length": 500.0,
      "epoch": 98.0,
      "grad_norm": 0.5169448852539062,
      "kl": 1.3162471055984497,
      "learning_rate": 4.379599518697444e-06,
      "loss": 0.0526,
      "reward": 3.255016803741455,
      "reward_std": 0.4777297377586365,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.875,
      "rewards/wrapped_driving_reward": 0.5050168633460999,
      "rewards/wrapped_format_reward": 0.875,
      "step": 490
    },
    {
      "completion_length": 500.0,
      "epoch": 98.2,
      "grad_norm": 0.553719699382782,
      "kl": 0.4232349097728729,
      "learning_rate": 4.375998874879585e-06,
      "loss": 0.0169,
      "reward": -2.375,
      "reward_std": 1.6007810831069946,
      "rewards/mpc_param_extraction_reward": 0.5,
      "rewards/mpc_param_name_reward": 0.5,
      "rewards/wrapped_driving_reward": -4.0,
      "rewards/wrapped_format_reward": 0.625,
      "step": 491
    },
    {
      "completion_length": 500.0,
      "epoch": 98.4,
      "grad_norm": 0.549466073513031,
      "kl": 0.9794742465019226,
      "learning_rate": 4.372389301972506e-06,
      "loss": 0.0392,
      "reward": 2.8539419174194336,
      "reward_std": 0.11372269690036774,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -0.14605820178985596,
      "rewards/wrapped_format_reward": 1.0,
      "step": 492
    },
    {
      "completion_length": 500.0,
      "epoch": 98.6,
      "grad_norm": 0.4947136342525482,
      "kl": 0.7255305051803589,
      "learning_rate": 4.368770817156493e-06,
      "loss": 0.029,
      "reward": 1.4876158237457275,
      "reward_std": 3.332230567932129,
      "rewards/mpc_param_extraction_reward": 0.75,
      "rewards/mpc_param_name_reward": 0.75,
      "rewards/wrapped_driving_reward": -0.3873841464519501,
      "rewards/wrapped_format_reward": 0.375,
      "step": 493
    },
    {
      "completion_length": 480.0,
      "epoch": 98.8,
      "grad_norm": 0.60732501745224,
      "kl": 0.8074931502342224,
      "learning_rate": 4.365143437654249e-06,
      "loss": 0.0323,
      "reward": 3.7022383213043213,
      "reward_std": 0.10065864771604538,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.96875,
      "rewards/wrapped_driving_reward": 0.7334883213043213,
      "rewards/wrapped_format_reward": 1.0,
      "step": 494
    },
    {
      "completion_length": 471.0,
      "epoch": 99.0,
      "grad_norm": 0.5607241988182068,
      "kl": 0.9837010502815247,
      "learning_rate": 4.3615071807308165e-06,
      "loss": 0.0393,
      "reward": 2.795694589614868,
      "reward_std": 0.325857937335968,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.04569460451602936,
      "rewards/wrapped_format_reward": 0.75,
      "step": 495
    },
    {
      "completion_length": 349.0,
      "epoch": 99.2,
      "grad_norm": 0.5761314034461975,
      "kl": 0.4112725853919983,
      "learning_rate": 4.357862063693486e-06,
      "loss": 0.0165,
      "reward": 3.6457560062408447,
      "reward_std": 0.12520428001880646,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.984375,
      "rewards/wrapped_driving_reward": 0.6613809466362,
      "rewards/wrapped_format_reward": 1.0,
      "step": 496
    },
    {
      "completion_length": 500.0,
      "epoch": 99.4,
      "grad_norm": 0.4828655421733856,
      "kl": 0.8910616636276245,
      "learning_rate": 4.354208103891723e-06,
      "loss": 0.0356,
      "reward": 3.4025442600250244,
      "reward_std": 0.550991415977478,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.96875,
      "rewards/wrapped_driving_reward": 0.6837941408157349,
      "rewards/wrapped_format_reward": 0.75,
      "step": 497
    },
    {
      "completion_length": 500.0,
      "epoch": 99.6,
      "grad_norm": 3.4703691005706787,
      "kl": 0.9643664360046387,
      "learning_rate": 4.350545318717081e-06,
      "loss": 0.0386,
      "reward": 2.7853288650512695,
      "reward_std": 0.7604644894599915,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": 0.03532897308468819,
      "rewards/wrapped_format_reward": 0.75,
      "step": 498
    },
    {
      "completion_length": 500.0,
      "epoch": 99.8,
      "grad_norm": 0.5264486074447632,
      "kl": 0.6139304041862488,
      "learning_rate": 4.3468737256031155e-06,
      "loss": 0.0246,
      "reward": 1.6845433712005615,
      "reward_std": 0.8301387429237366,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 1.0,
      "rewards/wrapped_driving_reward": -1.1904566287994385,
      "rewards/wrapped_format_reward": 0.875,
      "step": 499
    },
    {
      "completion_length": 419.0,
      "epoch": 100.0,
      "grad_norm": 0.5735057592391968,
      "kl": 0.41647979617118835,
      "learning_rate": 4.34319334202531e-06,
      "loss": 0.0167,
      "reward": 2.6967225074768066,
      "reward_std": 0.0777384340763092,
      "rewards/mpc_param_extraction_reward": 1.0,
      "rewards/mpc_param_name_reward": 0.9437500238418579,
      "rewards/wrapped_driving_reward": -0.24702748656272888,
      "rewards/wrapped_format_reward": 1.0,
      "step": 500
    }
  ],
  "logging_steps": 1,
  "max_steps": 1600,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 320,
  "save_steps": 250,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}