diff --git "a/checkpoint-1500/trainer_state.json" "b/checkpoint-1500/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/checkpoint-1500/trainer_state.json"
@@ -0,0 +1,22534 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 300.0,
+  "eval_steps": 500,
+  "global_step": 1500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "completion_length": 500.0,
+      "epoch": 0.2,
+      "grad_norm": 144.23707580566406,
+      "kl": 51.48179244995117,
+      "learning_rate": 3.1250000000000005e-08,
+      "loss": 2.0593,
+      "reward": 0.9761996865272522,
+      "reward_std": 3.3251326084136963,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.148800253868103,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 0.4,
+      "grad_norm": 974.2139892578125,
+      "kl": 216.24957275390625,
+      "learning_rate": 6.250000000000001e-08,
+      "loss": 8.65,
+      "reward": -3.75,
+      "reward_std": 0.5,
+      "rewards/mpc_param_extraction_reward": 0.0,
+      "rewards/mpc_param_name_reward": 0.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 2
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 0.6,
+      "grad_norm": 41697.91015625,
+      "kl": 3837.41943359375,
+      "learning_rate": 9.375e-08,
+      "loss": 153.4967,
+      "reward": -0.7961921691894531,
+      "reward_std": 3.700653076171875,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.4375,
+      "rewards/wrapped_driving_reward": -1.7336921691894531,
+      "rewards/wrapped_format_reward": 0.0,
+      "step": 3
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 0.8,
+      "grad_norm": 10122959.0,
+      "kl": 511094.90625,
+      "learning_rate": 1.2500000000000002e-07,
+      "loss": 20443.7988,
+      "reward": -2.338921546936035,
+      "reward_std": 3.322157144546509,
+      "rewards/mpc_param_extraction_reward": 0.25,
+      "rewards/mpc_param_name_reward": 0.25,
+      "rewards/wrapped_driving_reward": -2.838921546936035,
+      "rewards/wrapped_format_reward": 0.0,
+      "step": 4
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 1.0,
+      "grad_norm": 100702232.0,
+      "kl": 5416315.0,
+      "learning_rate": 1.5625e-07,
+      "loss": 216652.5938,
+      "reward": -0.16450506448745728,
+      "reward_std": 3.8515079021453857,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.6645050048828125,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 5
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 1.2,
+      "grad_norm": 17.33243751525879,
+      "kl": 7.672175884246826,
+      "learning_rate": 1.875e-07,
+      "loss": 0.3069,
+      "reward": 0.9893605709075928,
+      "reward_std": 1.5257619619369507,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.3856394290924072,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 6
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 1.4,
+      "grad_norm": 70.34513092041016,
+      "kl": 17.917146682739258,
+      "learning_rate": 2.1875e-07,
+      "loss": 0.7167,
+      "reward": 1.2267450094223022,
+      "reward_std": 3.4932949542999268,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.5232549905776978,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 7
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 1.6,
+      "grad_norm": 69448.7734375,
+      "kl": 9786.2802734375,
+      "learning_rate": 2.5000000000000004e-07,
+      "loss": 391.4512,
+      "reward": -0.967779815196991,
+      "reward_std": 3.5180184841156006,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -2.2177798748016357,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 8
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 1.8,
+      "grad_norm": 7205431.5,
+      "kl": 363326.15625,
+      "learning_rate": 2.8125e-07,
+      "loss": 14533.0439,
+      "reward": -0.4434952139854431,
+      "reward_std": 4.112156867980957,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.5684951543807983,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 9
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 2.0,
+      "grad_norm": 1344.39306640625,
+      "kl": 182.57179260253906,
+      "learning_rate": 3.125e-07,
+      "loss": 7.3029,
+      "reward": -0.5283111929893494,
+      "reward_std": 3.7256903648376465,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.6533112525939941,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 10
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 2.2,
+      "grad_norm": 663.510498046875,
+      "kl": 125.65680694580078,
+      "learning_rate": 3.4375000000000004e-07,
+      "loss": 5.0263,
+      "reward": -2.449397563934326,
+      "reward_std": 3.1012051105499268,
+      "rewards/mpc_param_extraction_reward": 0.25,
+      "rewards/mpc_param_name_reward": 0.25,
+      "rewards/wrapped_driving_reward": -2.949397563934326,
+      "rewards/wrapped_format_reward": 0.0,
+      "step": 11
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 2.4,
+      "grad_norm": 31.331226348876953,
+      "kl": 10.755382537841797,
+      "learning_rate": 3.75e-07,
+      "loss": 0.4302,
+      "reward": -2.1561226844787598,
+      "reward_std": 3.36269211769104,
+      "rewards/mpc_param_extraction_reward": 0.25,
+      "rewards/mpc_param_name_reward": 0.25,
+      "rewards/wrapped_driving_reward": -2.7811226844787598,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 12
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 2.6,
+      "grad_norm": 10.003079414367676,
+      "kl": 3.8625946044921875,
+      "learning_rate": 4.0625000000000003e-07,
+      "loss": 0.1545,
+      "reward": -2.1760454177856445,
+      "reward_std": 3.647908926010132,
+      "rewards/mpc_param_extraction_reward": 0.25,
+      "rewards/mpc_param_name_reward": 0.25,
+      "rewards/wrapped_driving_reward": -2.8010454177856445,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 13
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 2.8,
+      "grad_norm": 62.2325325012207,
+      "kl": 13.510702133178711,
+      "learning_rate": 4.375e-07,
+      "loss": 0.5404,
+      "reward": -2.294018030166626,
+      "reward_std": 3.0876402854919434,
+      "rewards/mpc_param_extraction_reward": 0.25,
+      "rewards/mpc_param_name_reward": 0.25,
+      "rewards/wrapped_driving_reward": -3.044018030166626,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 14
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 3.0,
+      "grad_norm": 96.1074447631836,
+      "kl": 10.679292678833008,
+      "learning_rate": 4.6875000000000006e-07,
+      "loss": 0.4272,
+      "reward": -0.6371059417724609,
+      "reward_std": 3.885227680206299,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.637105941772461,
+      "rewards/wrapped_format_reward": 0.0,
+      "step": 15
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 3.2,
+      "grad_norm": 1650.8782958984375,
+      "kl": 208.3596954345703,
+      "learning_rate": 5.000000000000001e-07,
+      "loss": 8.3344,
+      "reward": -4.0,
+      "reward_std": 0.0,
+      "rewards/mpc_param_extraction_reward": 0.0,
+      "rewards/mpc_param_name_reward": 0.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.0,
+      "step": 16
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 3.4,
+      "grad_norm": 17.093393325805664,
+      "kl": 5.905396461486816,
+      "learning_rate": 5.3125e-07,
+      "loss": 0.2362,
+      "reward": -2.4352118968963623,
+      "reward_std": 2.806159257888794,
+      "rewards/mpc_param_extraction_reward": 0.25,
+      "rewards/mpc_param_name_reward": 0.25,
+      "rewards/wrapped_driving_reward": -3.060211658477783,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 17
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 3.6,
+      "grad_norm": 78087.4140625,
+      "kl": 7675.3564453125,
+      "learning_rate": 5.625e-07,
+      "loss": 307.0142,
+      "reward": -0.4786604046821594,
+      "reward_std": 4.071903228759766,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.8536603450775146,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 18
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 3.8,
+      "grad_norm": 2062.067626953125,
+      "kl": 105.56303405761719,
+      "learning_rate": 5.9375e-07,
+      "loss": 4.2225,
+      "reward": -0.4283701777458191,
+      "reward_std": 4.145442008972168,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.8033702373504639,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 19
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 4.0,
+      "grad_norm": 513992.65625,
+      "kl": 39077.08984375,
+      "learning_rate": 6.25e-07,
+      "loss": 1563.0836,
+      "reward": -2.0327651500701904,
+      "reward_std": 3.3016297817230225,
+      "rewards/mpc_param_extraction_reward": 0.25,
+      "rewards/mpc_param_name_reward": 0.25,
+      "rewards/wrapped_driving_reward": -2.7827651500701904,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 20
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 4.2,
+      "grad_norm": 271.5398254394531,
+      "kl": 37.08869934082031,
+      "learning_rate": 6.562500000000001e-07,
+      "loss": 1.4835,
+      "reward": -3.5,
+      "reward_std": 0.5773502588272095,
+      "rewards/mpc_param_extraction_reward": 0.0,
+      "rewards/mpc_param_name_reward": 0.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 21
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 4.4,
+      "grad_norm": 33774.453125,
+      "kl": 4115.1591796875,
+      "learning_rate": 6.875000000000001e-07,
+      "loss": 164.6064,
+      "reward": -1.1355788707733154,
+      "reward_std": 3.42315673828125,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -2.5105788707733154,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 22
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 4.6,
+      "grad_norm": 52.09832000732422,
+      "kl": 14.069100379943848,
+      "learning_rate": 7.1875e-07,
+      "loss": 0.5628,
+      "reward": 0.9047523736953735,
+      "reward_std": 3.2798702716827393,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.5952475666999817,
+      "rewards/wrapped_format_reward": 0.0,
+      "step": 23
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 4.8,
+      "grad_norm": 75.83870697021484,
+      "kl": 16.262989044189453,
+      "learning_rate": 7.5e-07,
+      "loss": 0.6505,
+      "reward": -0.5572073459625244,
+      "reward_std": 2.9710958003997803,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -2.0572073459625244,
+      "rewards/wrapped_format_reward": 0.0,
+      "step": 24
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 5.0,
+      "grad_norm": 8.57257080078125,
+      "kl": 3.3865182399749756,
+      "learning_rate": 7.8125e-07,
+      "loss": 0.1355,
+      "reward": -2.110412120819092,
+      "reward_std": 3.7791755199432373,
+      "rewards/mpc_param_extraction_reward": 0.25,
+      "rewards/mpc_param_name_reward": 0.25,
+      "rewards/wrapped_driving_reward": -2.860412120819092,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 25
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 5.2,
+      "grad_norm": 978403.0,
+      "kl": 89647.2578125,
+      "learning_rate": 8.125000000000001e-07,
+      "loss": 3585.8899,
+      "reward": -0.2944529056549072,
+      "reward_std": 4.2804718017578125,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.6694529056549072,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 26
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 5.4,
+      "grad_norm": 168.73036193847656,
+      "kl": 29.724079132080078,
+      "learning_rate": 8.437500000000001e-07,
+      "loss": 1.189,
+      "reward": -0.6913368701934814,
+      "reward_std": 3.5795211791992188,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -2.0663368701934814,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 27
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 5.6,
+      "grad_norm": 75324.5234375,
+      "kl": 7936.74267578125,
+      "learning_rate": 8.75e-07,
+      "loss": 317.4697,
+      "reward": 0.9355948567390442,
+      "reward_std": 3.3464736938476562,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.6894051432609558,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 28
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 5.8,
+      "grad_norm": 469.9671630859375,
+      "kl": 71.2878189086914,
+      "learning_rate": 9.0625e-07,
+      "loss": 2.8515,
+      "reward": -2.547950267791748,
+      "reward_std": 2.904099464416504,
+      "rewards/mpc_param_extraction_reward": 0.25,
+      "rewards/mpc_param_name_reward": 0.25,
+      "rewards/wrapped_driving_reward": -3.047950267791748,
+      "rewards/wrapped_format_reward": 0.0,
+      "step": 29
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 6.0,
+      "grad_norm": 36.652645111083984,
+      "kl": 13.148932456970215,
+      "learning_rate": 9.375000000000001e-07,
+      "loss": 0.526,
+      "reward": -3.375,
+      "reward_std": 0.9464846849441528,
+      "rewards/mpc_param_extraction_reward": 0.25,
+      "rewards/mpc_param_name_reward": 0.25,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 30
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 6.2,
+      "grad_norm": 30.199684143066406,
+      "kl": 9.480849266052246,
+      "learning_rate": 9.6875e-07,
+      "loss": 0.3792,
+      "reward": -2.0530290603637695,
+      "reward_std": 3.2615222930908203,
+      "rewards/mpc_param_extraction_reward": 0.25,
+      "rewards/mpc_param_name_reward": 0.25,
+      "rewards/wrapped_driving_reward": -2.9280290603637695,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 31
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 6.4,
+      "grad_norm": 78.3298568725586,
+      "kl": 26.2161865234375,
+      "learning_rate": 1.0000000000000002e-06,
+      "loss": 1.0486,
+      "reward": 1.33430814743042,
+      "reward_std": 3.5583572387695312,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.41569197177886963,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 32
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 6.6,
+      "grad_norm": 19.472774505615234,
+      "kl": 7.02009391784668,
+      "learning_rate": 1.03125e-06,
+      "loss": 0.2808,
+      "reward": -0.6657888889312744,
+      "reward_std": 3.853987216949463,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.6657888889312744,
+      "rewards/wrapped_format_reward": 0.0,
+      "step": 33
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 6.8,
+      "grad_norm": 65.95396423339844,
+      "kl": 18.14912223815918,
+      "learning_rate": 1.0625e-06,
+      "loss": 0.726,
+      "reward": -0.8661626577377319,
+      "reward_std": 3.6189870834350586,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -2.1161625385284424,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 34
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 7.0,
+      "grad_norm": 1.4008393287658691,
+      "kl": 0.8411699533462524,
+      "learning_rate": 1.0937500000000001e-06,
+      "loss": 0.0336,
+      "reward": -0.38928359746932983,
+      "reward_std": 4.171046257019043,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.6392836570739746,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 35
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 7.2,
+      "grad_norm": 7.80991268157959,
+      "kl": 4.218427658081055,
+      "learning_rate": 1.125e-06,
+      "loss": 0.1687,
+      "reward": -1.1417465209960938,
+      "reward_std": 3.381352186203003,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -2.1417465209960938,
+      "rewards/wrapped_format_reward": 0.0,
+      "step": 36
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 7.4,
+      "grad_norm": 4.478097438812256,
+      "kl": 2.112290143966675,
+      "learning_rate": 1.1562500000000002e-06,
+      "loss": 0.0845,
+      "reward": -2.102426528930664,
+      "reward_std": 3.1373467445373535,
+      "rewards/mpc_param_extraction_reward": 0.25,
+      "rewards/mpc_param_name_reward": 0.25,
+      "rewards/wrapped_driving_reward": -2.852426528930664,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 37
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 7.6,
+      "grad_norm": 9.182758331298828,
+      "kl": 4.63405179977417,
+      "learning_rate": 1.1875e-06,
+      "loss": 0.1854,
+      "reward": -2.2051236629486084,
+      "reward_std": 3.2649383544921875,
+      "rewards/mpc_param_extraction_reward": 0.25,
+      "rewards/mpc_param_name_reward": 0.25,
+      "rewards/wrapped_driving_reward": -2.9551236629486084,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 38
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 7.8,
+      "grad_norm": 203072.515625,
+      "kl": 20056.82421875,
+      "learning_rate": 1.21875e-06,
+      "loss": 802.2729,
+      "reward": -2.125791311264038,
+      "reward_std": 3.0907511711120605,
+      "rewards/mpc_param_extraction_reward": 0.25,
+      "rewards/mpc_param_name_reward": 0.25,
+      "rewards/wrapped_driving_reward": -3.000791311264038,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 39
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 8.0,
+      "grad_norm": 5.843477725982666,
+      "kl": 3.436691999435425,
+      "learning_rate": 1.25e-06,
+      "loss": 0.1375,
+      "reward": -0.9351435899734497,
+      "reward_std": 3.608586072921753,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -2.3101437091827393,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 40
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 8.2,
+      "grad_norm": 7742784.5,
+      "kl": 650933.375,
+      "learning_rate": 1.28125e-06,
+      "loss": 26037.334,
+      "reward": 1.5518302917480469,
+      "reward_std": 3.7290942668914795,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.4481697678565979,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 41
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 8.4,
+      "grad_norm": 8.937725067138672,
+      "kl": 3.0701639652252197,
+      "learning_rate": 1.3125000000000001e-06,
+      "loss": 0.1228,
+      "reward": -2.1904397010803223,
+      "reward_std": 3.2942306995391846,
+      "rewards/mpc_param_extraction_reward": 0.25,
+      "rewards/mpc_param_name_reward": 0.25,
+      "rewards/wrapped_driving_reward": -3.0654397010803223,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 42
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 8.6,
+      "grad_norm": 14.36681079864502,
+      "kl": 5.88793420791626,
+      "learning_rate": 1.34375e-06,
+      "loss": 0.2355,
+      "reward": 0.6519123315811157,
+      "reward_std": 3.105113983154297,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.973087728023529,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 43
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 8.8,
+      "grad_norm": 180.1724395751953,
+      "kl": 33.255760192871094,
+      "learning_rate": 1.3750000000000002e-06,
+      "loss": 1.3302,
+      "reward": -0.6511552333831787,
+      "reward_std": 3.9030916690826416,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.7761552333831787,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 44
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 9.0,
+      "grad_norm": 32.81709671020508,
+      "kl": 6.7791428565979,
+      "learning_rate": 1.40625e-06,
+      "loss": 0.2712,
+      "reward": -3.875,
+      "reward_std": 0.25,
+      "rewards/mpc_param_extraction_reward": 0.0,
+      "rewards/mpc_param_name_reward": 0.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 45
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 9.2,
+      "grad_norm": 4151.640625,
+      "kl": 350.8572082519531,
+      "learning_rate": 1.4375e-06,
+      "loss": 14.0343,
+      "reward": -1.1032943725585938,
+      "reward_std": 3.3453028202056885,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -2.1032943725585938,
+      "rewards/wrapped_format_reward": 0.0,
+      "step": 46
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 9.4,
+      "grad_norm": 15.409821510314941,
+      "kl": 5.346187114715576,
+      "learning_rate": 1.4687500000000001e-06,
+      "loss": 0.2138,
+      "reward": -1.1791430711746216,
+      "reward_std": 2.8540005683898926,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -2.929143190383911,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 47
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 9.6,
+      "grad_norm": 144.20443725585938,
+      "kl": 23.608051300048828,
+      "learning_rate": 1.5e-06,
+      "loss": 0.9443,
+      "reward": -2.530029058456421,
+      "reward_std": 2.939941883087158,
+      "rewards/mpc_param_extraction_reward": 0.25,
+      "rewards/mpc_param_name_reward": 0.20000000298023224,
+      "rewards/wrapped_driving_reward": -2.9800291061401367,
+      "rewards/wrapped_format_reward": 0.0,
+      "step": 48
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 9.8,
+      "grad_norm": 7.547443866729736,
+      "kl": 3.824962615966797,
+      "learning_rate": 1.5312500000000002e-06,
+      "loss": 0.153,
+      "reward": -0.7816690802574158,
+      "reward_std": 3.7201201915740967,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.906669020652771,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 49
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 10.0,
+      "grad_norm": 77.945556640625,
+      "kl": 16.699840545654297,
+      "learning_rate": 1.5625e-06,
+      "loss": 0.668,
+      "reward": -0.2709696292877197,
+      "reward_std": 4.022421360015869,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.6459696292877197,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 50
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 10.2,
+      "grad_norm": 11738.6953125,
+      "kl": 798.2957763671875,
+      "learning_rate": 1.59375e-06,
+      "loss": 31.9318,
+      "reward": -2.244354724884033,
+      "reward_std": 3.1866860389709473,
+      "rewards/mpc_param_extraction_reward": 0.25,
+      "rewards/mpc_param_name_reward": 0.25,
+      "rewards/wrapped_driving_reward": -2.869354724884033,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 51
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 10.4,
+      "grad_norm": 14.563969612121582,
+      "kl": 5.301497936248779,
+      "learning_rate": 1.6250000000000001e-06,
+      "loss": 0.2121,
+      "reward": 2.282280445098877,
+      "reward_std": 0.7978482246398926,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.21771956980228424,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 52
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 10.6,
+      "grad_norm": 6.280083656311035,
+      "kl": 3.3535187244415283,
+      "learning_rate": 1.65625e-06,
+      "loss": 0.1341,
+      "reward": 0.2996126413345337,
+      "reward_std": 2.949772357940674,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.3253873586654663,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 53
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 10.8,
+      "grad_norm": 223484.046875,
+      "kl": 25810.041015625,
+      "learning_rate": 1.6875000000000001e-06,
+      "loss": 1032.4015,
+      "reward": -0.344623327255249,
+      "reward_std": 3.944869041442871,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.719623327255249,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 54
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 11.0,
+      "grad_norm": 593881.0,
+      "kl": 99004.265625,
+      "learning_rate": 1.71875e-06,
+      "loss": 3960.1709,
+      "reward": -3.875,
+      "reward_std": 0.25,
+      "rewards/mpc_param_extraction_reward": 0.0,
+      "rewards/mpc_param_name_reward": 0.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 55
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 11.2,
+      "grad_norm": 6.934082508087158,
+      "kl": 2.1647584438323975,
+      "learning_rate": 1.75e-06,
+      "loss": 0.0866,
+      "reward": -1.2983622550964355,
+      "reward_std": 2.1767022609710693,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -3.1733622550964355,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 56
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 11.4,
+      "grad_norm": 11660.44921875,
+      "kl": 1039.497802734375,
+      "learning_rate": 1.78125e-06,
+      "loss": 41.5799,
+      "reward": 1.111867070198059,
+      "reward_std": 3.4727301597595215,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.6381329298019409,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 57
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 11.6,
+      "grad_norm": 116.61476135253906,
+      "kl": 30.001558303833008,
+      "learning_rate": 1.8125e-06,
+      "loss": 1.2001,
+      "reward": -3.875,
+      "reward_std": 0.25,
+      "rewards/mpc_param_extraction_reward": 0.0,
+      "rewards/mpc_param_name_reward": 0.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 58
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 11.8,
+      "grad_norm": 7.750627040863037,
+      "kl": 3.2049598693847656,
+      "learning_rate": 1.8437500000000003e-06,
+      "loss": 0.1282,
+      "reward": -3.875,
+      "reward_std": 0.25,
+      "rewards/mpc_param_extraction_reward": 0.0,
+      "rewards/mpc_param_name_reward": 0.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 59
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 12.0,
+      "grad_norm": 6.483101844787598,
+      "kl": 2.8182482719421387,
+      "learning_rate": 1.8750000000000003e-06,
+      "loss": 0.1127,
+      "reward": 0.4944196343421936,
+      "reward_std": 2.8224055767059326,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.3805804252624512,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 60
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 12.2,
+      "grad_norm": 2.41937255859375,
+      "kl": 1.5243698358535767,
+      "learning_rate": 1.90625e-06,
+      "loss": 0.061,
+      "reward": -1.5549238920211792,
+      "reward_std": 3.3336286544799805,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -2.9299237728118896,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 61
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 12.4,
+      "grad_norm": 1.8835395574569702,
+      "kl": 1.3928029537200928,
+      "learning_rate": 1.9375e-06,
+      "loss": 0.0557,
+      "reward": 0.5334538817405701,
+      "reward_std": 3.0702548027038574,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9665461778640747,
+      "rewards/wrapped_format_reward": 0.0,
+      "step": 62
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 12.6,
+      "grad_norm": 82.16962432861328,
+      "kl": 9.488784790039062,
+      "learning_rate": 1.96875e-06,
+      "loss": 0.3796,
+      "reward": -0.4010847806930542,
+      "reward_std": 4.155675411224365,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.7760847806930542,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 63
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 12.8,
+      "grad_norm": 1.6330454349517822,
+      "kl": 0.7770444750785828,
+      "learning_rate": 2.0000000000000003e-06,
+      "loss": 0.0311,
+      "reward": -1.0365194082260132,
+      "reward_std": 3.1399788856506348,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.46875,
+      "rewards/wrapped_driving_reward": -2.2552695274353027,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 64
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 13.0,
+      "grad_norm": 226.30535888671875,
+      "kl": 45.10585021972656,
+      "learning_rate": 2.0312500000000002e-06,
+      "loss": 1.8042,
+      "reward": -1.9248578548431396,
+      "reward_std": 3.5153682231903076,
+      "rewards/mpc_param_extraction_reward": 0.25,
+      "rewards/mpc_param_name_reward": 0.25,
+      "rewards/wrapped_driving_reward": -2.7998578548431396,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 65
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 13.2,
+      "grad_norm": 62.87789535522461,
+      "kl": 7.880831718444824,
+      "learning_rate": 2.0625e-06,
+      "loss": 0.3152,
+      "reward": 3.1509861946105957,
+      "reward_std": 0.29935261607170105,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.987500011920929,
+      "rewards/wrapped_driving_reward": 0.5384860038757324,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 66
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 13.4,
+      "grad_norm": 41.82035446166992,
+      "kl": 9.357061386108398,
+      "learning_rate": 2.09375e-06,
+      "loss": 0.3743,
+      "reward": -2.576190948486328,
+      "reward_std": 2.8476178646087646,
+      "rewards/mpc_param_extraction_reward": 0.25,
+      "rewards/mpc_param_name_reward": 0.25,
+      "rewards/wrapped_driving_reward": -3.201190948486328,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 67
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 13.6,
+      "grad_norm": 4.275550365447998,
+      "kl": 2.5297634601593018,
+      "learning_rate": 2.125e-06,
+      "loss": 0.1012,
+      "reward": -2.036945104598999,
+      "reward_std": 3.926109790802002,
+      "rewards/mpc_param_extraction_reward": 0.25,
+      "rewards/mpc_param_name_reward": 0.25,
+      "rewards/wrapped_driving_reward": -2.786945104598999,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 68
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 13.8,
+      "grad_norm": 12.396137237548828,
+      "kl": 3.0427801609039307,
+      "learning_rate": 2.1562500000000003e-06,
+      "loss": 0.1217,
+      "reward": 0.7283755540847778,
+      "reward_std": 3.323927879333496,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.2716244459152222,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 69
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 14.0,
+      "grad_norm": 1.5282281637191772,
+      "kl": 1.092595100402832,
+      "learning_rate": 2.1875000000000002e-06,
+      "loss": 0.0437,
+      "reward": -3.2356114387512207,
+      "reward_std": 1.528777003288269,
+      "rewards/mpc_param_extraction_reward": 0.25,
+      "rewards/mpc_param_name_reward": 0.25,
+      "rewards/wrapped_driving_reward": -3.8606114387512207,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 70
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 14.2,
+      "grad_norm": 1.2480015754699707,
+      "kl": 0.7834239602088928,
+      "learning_rate": 2.21875e-06,
+      "loss": 0.0313,
+      "reward": -2.0355887413024902,
+      "reward_std": 3.270659923553467,
+      "rewards/mpc_param_extraction_reward": 0.25,
+      "rewards/mpc_param_name_reward": 0.25,
+      "rewards/wrapped_driving_reward": -2.7855887413024902,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 71
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 14.4,
+      "grad_norm": 80.85037231445312,
+      "kl": 9.716327667236328,
+      "learning_rate": 2.25e-06,
+      "loss": 0.3887,
+      "reward": -3.375,
+      "reward_std": 1.25,
+      "rewards/mpc_param_extraction_reward": 0.25,
+      "rewards/mpc_param_name_reward": 0.25,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 72
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 14.6,
+      "grad_norm": 1.1875276565551758,
+      "kl": 0.9156450629234314,
+      "learning_rate": 2.28125e-06,
+      "loss": 0.0366,
+      "reward": 0.398318886756897,
+      "reward_std": 2.9533674716949463,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.3516812324523926,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 73
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 14.8,
+      "grad_norm": 5.200348854064941,
+      "kl": 2.256690502166748,
+      "learning_rate": 2.3125000000000003e-06,
+      "loss": 0.0903,
+      "reward": -0.17332077026367188,
+      "reward_std": 4.147373199462891,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.47727274894714355,
+      "rewards/wrapped_driving_reward": -1.650593638420105,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 74
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 15.0,
+      "grad_norm": 7008.42626953125,
+      "kl": 815.8104248046875,
+      "learning_rate": 2.3437500000000002e-06,
+      "loss": 32.6324,
+      "reward": -2.0281713008880615,
+      "reward_std": 3.3107235431671143,
+      "rewards/mpc_param_extraction_reward": 0.25,
+      "rewards/mpc_param_name_reward": 0.25,
+      "rewards/wrapped_driving_reward": -2.9031713008880615,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 75
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 15.2,
+      "grad_norm": 0.9020519852638245,
+      "kl": 0.9176801443099976,
+      "learning_rate": 2.375e-06,
+      "loss": 0.0367,
+      "reward": -2.1365058422088623,
+      "reward_std": 3.0964157581329346,
+      "rewards/mpc_param_extraction_reward": 0.25,
+      "rewards/mpc_param_name_reward": 0.25,
+      "rewards/wrapped_driving_reward": -3.1365058422088623,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 76
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 15.4,
+      "grad_norm": 2.2519280910491943,
+      "kl": 0.8236393332481384,
+      "learning_rate": 2.40625e-06,
+      "loss": 0.0329,
+      "reward": -1.22861909866333,
+      "reward_std": 2.9241857528686523,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -2.353619337081909,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 77
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 15.6,
+      "grad_norm": 1.5832031965255737,
+      "kl": 0.7527546286582947,
+      "learning_rate": 2.4375e-06,
+      "loss": 0.0301,
+      "reward": -0.6693365573883057,
+      "reward_std": 3.860503911972046,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.7943366765975952,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 78
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 15.8,
+      "grad_norm": 1.108726143836975,
+      "kl": 1.0248883962631226,
+      "learning_rate": 2.4687500000000003e-06,
+      "loss": 0.041,
+      "reward": -3.125,
+      "reward_std": 1.4361406564712524,
+      "rewards/mpc_param_extraction_reward": 0.25,
+      "rewards/mpc_param_name_reward": 0.25,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 79
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 16.0,
+      "grad_norm": 1.0169743299484253,
+      "kl": 0.7592311501502991,
+      "learning_rate": 2.5e-06,
+      "loss": 0.0304,
+      "reward": 1.145168423652649,
+      "reward_std": 3.56965708732605,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.7298316359519958,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 80
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 16.2,
+      "grad_norm": 0.7822604179382324,
+      "kl": 0.560085117816925,
+      "learning_rate": 2.53125e-06,
+      "loss": 0.0224,
+      "reward": 1.2056835889816284,
+      "reward_std": 3.5178937911987305,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.6693164110183716,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 81
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 16.4,
+      "grad_norm": 0.8175077438354492,
+      "kl": 0.5752599239349365,
+      "learning_rate": 2.5625e-06,
+      "loss": 0.023,
+      "reward": 2.9252407550811768,
+      "reward_std": 0.7892647385597229,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.3002408742904663,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 82
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 16.6,
+      "grad_norm": 6.47392463684082,
+      "kl": 2.1055572032928467,
+      "learning_rate": 2.5937500000000004e-06,
+      "loss": 0.0842,
+      "reward": -2.7708332538604736,
+      "reward_std": 1.4678263664245605,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.4791666865348816,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 83
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 16.8,
+      "grad_norm": 0.9220647811889648,
+      "kl": 0.7157851457595825,
+      "learning_rate": 2.6250000000000003e-06,
+      "loss": 0.0286,
+      "reward": -2.987729549407959,
+      "reward_std": 1.3781793117523193,
+      "rewards/mpc_param_extraction_reward": 0.25,
+      "rewards/mpc_param_name_reward": 0.25,
+      "rewards/wrapped_driving_reward": -3.987729549407959,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 84
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 17.0,
+      "grad_norm": 62.959754943847656,
+      "kl": 11.148348808288574,
+      "learning_rate": 2.65625e-06,
+      "loss": 0.4459,
+      "reward": -3.875,
+      "reward_std": 0.25,
+      "rewards/mpc_param_extraction_reward": 0.0,
+      "rewards/mpc_param_name_reward": 0.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 85
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 17.2,
+      "grad_norm": 1.0555181503295898,
+      "kl": 0.7983899712562561,
+      "learning_rate": 2.6875e-06,
+      "loss": 0.0319,
+      "reward": -0.34822893142700195,
+      "reward_std": 1.868666648864746,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -2.973228931427002,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 86
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 17.4,
+      "grad_norm": 5.494264602661133,
+      "kl": 1.303008794784546,
+      "learning_rate": 2.71875e-06,
+      "loss": 0.0521,
+      "reward": -1.6160635948181152,
+      "reward_std": 3.132638931274414,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -2.7410635948181152,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 87
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 17.6,
+      "grad_norm": 4.55032205581665,
+      "kl": 0.8609241247177124,
+      "learning_rate": 2.7500000000000004e-06,
+      "loss": 0.0344,
+      "reward": -0.07164722681045532,
+      "reward_std": 2.4449591636657715,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7045454978942871,
+      "rewards/wrapped_driving_reward": -1.9011927843093872,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 88
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 17.8,
+      "grad_norm": 0.9195135831832886,
+      "kl": 0.683874785900116,
+      "learning_rate": 2.7812500000000003e-06,
+      "loss": 0.0274,
+      "reward": -0.11890482902526855,
+      "reward_std": 3.1548802852630615,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.699999988079071,
+      "rewards/wrapped_driving_reward": -2.0689048767089844,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 89
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 18.0,
+      "grad_norm": 33.38914108276367,
+      "kl": 7.705580234527588,
+      "learning_rate": 2.8125e-06,
+      "loss": 0.3082,
+      "reward": 1.586517572402954,
+      "reward_std": 3.729795217514038,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.4134823679924011,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 90
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 18.2,
+      "grad_norm": 57.7512092590332,
+      "kl": 6.441009998321533,
+      "learning_rate": 2.84375e-06,
+      "loss": 0.2576,
+      "reward": -0.8143091201782227,
+      "reward_std": 3.7249650955200195,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.4444444477558136,
+      "rewards/wrapped_driving_reward": -2.008753776550293,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 91
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 18.4,
+      "grad_norm": 1.0778491497039795,
+      "kl": 0.7857025265693665,
+      "learning_rate": 2.875e-06,
+      "loss": 0.0314,
+      "reward": -0.4593994617462158,
+      "reward_std": 3.805197238922119,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.8343994617462158,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 92
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 18.6,
+      "grad_norm": 1.1437242031097412,
+      "kl": 0.5162321925163269,
+      "learning_rate": 2.9062500000000003e-06,
+      "loss": 0.0206,
+      "reward": -1.5748236179351807,
+      "reward_std": 3.3948116302490234,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -2.8248236179351807,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 93
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 18.8,
+      "grad_norm": 0.782410204410553,
+      "kl": 0.4215336740016937,
+      "learning_rate": 2.9375000000000003e-06,
+      "loss": 0.0169,
+      "reward": -1.7746977806091309,
+      "reward_std": 3.3859715461730957,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.4375,
+      "rewards/wrapped_driving_reward": -2.962197780609131,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 94
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 19.0,
+      "grad_norm": 4.5705718994140625,
+      "kl": 2.0152359008789062,
+      "learning_rate": 2.96875e-06,
+      "loss": 0.0806,
+      "reward": 0.6353222131729126,
+      "reward_std": 3.1809890270233154,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.1146776676177979,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 95
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 19.2,
+      "grad_norm": 0.7173673510551453,
+      "kl": 0.5042878985404968,
+      "learning_rate": 3e-06,
+      "loss": 0.0202,
+      "reward": 3.3471288681030273,
+      "reward_std": 0.31114432215690613,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.722128689289093,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 96
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 19.4,
+      "grad_norm": 2.924496650695801,
+      "kl": 0.9709882736206055,
+      "learning_rate": 3.03125e-06,
+      "loss": 0.0388,
+      "reward": 1.2860008478164673,
+      "reward_std": 2.1523053646087646,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9166666865348816,
+      "rewards/wrapped_driving_reward": -1.1306657791137695,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 97
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 19.6,
+      "grad_norm": 1.3449220657348633,
+      "kl": 0.776192843914032,
+      "learning_rate": 3.0625000000000003e-06,
+      "loss": 0.031,
+      "reward": -1.2475244998931885,
+      "reward_std": 3.184887170791626,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -2.2475244998931885,
+      "rewards/wrapped_format_reward": 0.0,
+      "step": 98
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 19.8,
+      "grad_norm": 16.095233917236328,
+      "kl": 1.6037352085113525,
+      "learning_rate": 3.0937500000000002e-06,
+      "loss": 0.0641,
+      "reward": 0.7092133164405823,
+      "reward_std": 3.2424275875091553,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.040786862373352,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 99
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 20.0,
+      "grad_norm": 1.065063714981079,
+      "kl": 0.6967657208442688,
+      "learning_rate": 3.125e-06,
+      "loss": 0.0279,
+      "reward": 1.58005690574646,
+      "reward_std": 3.7297377586364746,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.41994309425354004,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 100
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 20.2,
+      "grad_norm": 0.7442240715026855,
+      "kl": 0.5057598352432251,
+      "learning_rate": 3.15625e-06,
+      "loss": 0.0202,
+      "reward": -0.4889770746231079,
+      "reward_std": 3.2432987689971924,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -2.1139769554138184,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 101
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 20.4,
+      "grad_norm": 0.6366997361183167,
+      "kl": 0.44367504119873047,
+      "learning_rate": 3.1875e-06,
+      "loss": 0.0177,
+      "reward": -2.3750693798065186,
+      "reward_std": 2.5939254760742188,
+      "rewards/mpc_param_extraction_reward": 0.25,
+      "rewards/mpc_param_name_reward": 0.25,
+      "rewards/wrapped_driving_reward": -3.2500693798065186,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 102
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 20.6,
+      "grad_norm": 2.0096611976623535,
+      "kl": 0.4689376652240753,
+      "learning_rate": 3.2187500000000003e-06,
+      "loss": 0.0188,
+      "reward": -0.5932518243789673,
+      "reward_std": 3.942629814147949,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.9682518243789673,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 103
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 20.8,
+      "grad_norm": 1.2003757953643799,
+      "kl": 0.3688035309314728,
+      "learning_rate": 3.2500000000000002e-06,
+      "loss": 0.0148,
+      "reward": 2.4348607063293457,
+      "reward_std": 1.402535080909729,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.824999988079071,
+      "rewards/wrapped_driving_reward": 0.1098608672618866,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 104
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 21.0,
+      "grad_norm": 1.0951755046844482,
+      "kl": 0.6130416393280029,
+      "learning_rate": 3.28125e-06,
+      "loss": 0.0245,
+      "reward": 1.912153720855713,
+      "reward_std": 2.661609649658203,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.4628463387489319,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 105
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 21.2,
+      "grad_norm": 0.7108362317085266,
+      "kl": 0.4603574275970459,
+      "learning_rate": 3.3125e-06,
+      "loss": 0.0184,
+      "reward": -0.09991639852523804,
+      "reward_std": 2.701847791671753,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.699999988079071,
+      "rewards/wrapped_driving_reward": -1.674916386604309,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 106
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 21.4,
+      "grad_norm": 0.6701599955558777,
+      "kl": 0.40188899636268616,
+      "learning_rate": 3.34375e-06,
+      "loss": 0.0161,
+      "reward": -0.8211934566497803,
+      "reward_std": 3.6706011295318604,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -2.0711934566497803,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 107
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 21.6,
+      "grad_norm": 2.1547489166259766,
+      "kl": 1.2839192152023315,
+      "learning_rate": 3.3750000000000003e-06,
+      "loss": 0.0514,
+      "reward": 1.2595562934875488,
+      "reward_std": 3.514036178588867,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.36544373631477356,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 108
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 21.8,
+      "grad_norm": 1.838152527809143,
+      "kl": 0.49252963066101074,
+      "learning_rate": 3.40625e-06,
+      "loss": 0.0197,
+      "reward": 1.507678508758545,
+      "reward_std": 3.675663948059082,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7272727489471436,
+      "rewards/wrapped_driving_reward": -0.46959418058395386,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 109
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 22.0,
+      "grad_norm": 0.7439582347869873,
+      "kl": 0.595367431640625,
+      "learning_rate": 3.4375e-06,
+      "loss": 0.0238,
+      "reward": -3.625,
+      "reward_std": 0.4787135720252991,
+      "rewards/mpc_param_extraction_reward": 0.0,
+      "rewards/mpc_param_name_reward": 0.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 110
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 22.2,
+      "grad_norm": 0.5864555835723877,
+      "kl": 0.3803044855594635,
+      "learning_rate": 3.46875e-06,
+      "loss": 0.0152,
+      "reward": 1.1622142791748047,
+      "reward_std": 3.198068618774414,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7272727489471436,
+      "rewards/wrapped_driving_reward": -1.0650583505630493,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 111
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 22.4,
+      "grad_norm": 4.526273250579834,
+      "kl": 0.46392467617988586,
+      "learning_rate": 3.5e-06,
+      "loss": 0.0186,
+      "reward": -0.7247750163078308,
+      "reward_std": 3.514815330505371,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -2.0997748374938965,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 112
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 22.6,
+      "grad_norm": 0.5704318881034851,
+      "kl": 0.31563645601272583,
+      "learning_rate": 3.5312500000000007e-06,
+      "loss": 0.0126,
+      "reward": -0.6594128608703613,
+      "reward_std": 3.574946880340576,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -2.0344128608703613,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 113
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 22.8,
+      "grad_norm": 0.671708881855011,
+      "kl": 0.44867807626724243,
+      "learning_rate": 3.5625e-06,
+      "loss": 0.0179,
+      "reward": 1.6787878274917603,
+      "reward_std": 0.9073445200920105,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.9462121725082397,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 114
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 23.0,
+      "grad_norm": 1.2352200746536255,
+      "kl": 0.46100977063179016,
+      "learning_rate": 3.59375e-06,
+      "loss": 0.0184,
+      "reward": -0.30863749980926514,
+      "reward_std": 3.9797427654266357,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.8086374998092651,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 115
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 23.2,
+      "grad_norm": 0.783157229423523,
+      "kl": 0.44468817114830017,
+      "learning_rate": 3.625e-06,
+      "loss": 0.0178,
+      "reward": 1.9851404428482056,
+      "reward_std": 1.0498998165130615,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.2648596167564392,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 116
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 23.4,
+      "grad_norm": 0.6096097826957703,
+      "kl": 0.36140069365501404,
+      "learning_rate": 3.65625e-06,
+      "loss": 0.0145,
+      "reward": -0.9730753898620605,
+      "reward_std": 2.724126100540161,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -3.0980753898620605,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 117
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 23.6,
+      "grad_norm": 2.4817147254943848,
+      "kl": 0.3356289267539978,
+      "learning_rate": 3.6875000000000007e-06,
+      "loss": 0.0134,
+      "reward": 2.9043874740600586,
+      "reward_std": 0.34505343437194824,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.5293872952461243,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 118
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 23.8,
+      "grad_norm": 1.3457905054092407,
+      "kl": 0.32610735297203064,
+      "learning_rate": 3.7187500000000006e-06,
+      "loss": 0.013,
+      "reward": -0.004844188690185547,
+      "reward_std": 3.559382915496826,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.7548441886901855,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 119
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 24.0,
+      "grad_norm": 0.8271002769470215,
+      "kl": 0.592341423034668,
+      "learning_rate": 3.7500000000000005e-06,
+      "loss": 0.0237,
+      "reward": 0.23737984895706177,
+      "reward_std": 2.8921873569488525,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.6000000238418579,
+      "rewards/wrapped_driving_reward": -1.2376201152801514,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 120
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 24.2,
+      "grad_norm": 1.575377106666565,
+      "kl": 0.31468361616134644,
+      "learning_rate": 3.78125e-06,
+      "loss": 0.0126,
+      "reward": 0.08798408508300781,
+      "reward_std": 3.364243984222412,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.9120157957077026,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 121
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 24.4,
+      "grad_norm": 1.548542857170105,
+      "kl": 0.7125066518783569,
+      "learning_rate": 3.8125e-06,
+      "loss": 0.0285,
+      "reward": 3.202035903930664,
+      "reward_std": 0.5515704154968262,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8958333134651184,
+      "rewards/wrapped_driving_reward": 0.6812027096748352,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 122
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 24.6,
+      "grad_norm": 0.6466585397720337,
+      "kl": 0.33141595125198364,
+      "learning_rate": 3.84375e-06,
+      "loss": 0.0133,
+      "reward": -0.8563422560691833,
+      "reward_std": 2.9308738708496094,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -2.981342315673828,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 123
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 24.8,
+      "grad_norm": 0.9053751826286316,
+      "kl": 0.3941192626953125,
+      "learning_rate": 3.875e-06,
+      "loss": 0.0158,
+      "reward": -0.9088470935821533,
+      "reward_std": 2.4116313457489014,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -2.6588470935821533,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 124
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 25.0,
+      "grad_norm": 0.7404253482818604,
+      "kl": 0.3537856936454773,
+      "learning_rate": 3.90625e-06,
+      "loss": 0.0142,
+      "reward": -0.08935052156448364,
+      "reward_std": 4.237273693084717,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.5893504619598389,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 125
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 25.2,
+      "grad_norm": 0.5974608659744263,
+      "kl": 0.31292691826820374,
+      "learning_rate": 3.9375e-06,
+      "loss": 0.0125,
+      "reward": -0.5100458860397339,
+      "reward_std": 3.746746063232422,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.8850458860397339,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 126
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 25.4,
+      "grad_norm": 0.9886866807937622,
+      "kl": 0.3266676068305969,
+      "learning_rate": 3.96875e-06,
+      "loss": 0.0131,
+      "reward": 3.5397558212280273,
+      "reward_std": 0.24529722332954407,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.7897558212280273,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 127
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 25.6,
+      "grad_norm": 0.6569087505340576,
+      "kl": 0.28314509987831116,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 0.0113,
+      "reward": -0.5303106904029846,
+      "reward_std": 4.0666303634643555,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.375,
+      "rewards/wrapped_driving_reward": -1.7803106307983398,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 128
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 25.8,
+      "grad_norm": 1.4679771661758423,
+      "kl": 0.4160246253013611,
+      "learning_rate": 4.031250000000001e-06,
+      "loss": 0.0166,
+      "reward": -0.5868573188781738,
+      "reward_std": 3.941485643386841,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.8368571996688843,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 129
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 26.0,
+      "grad_norm": 0.5985941290855408,
+      "kl": 0.31736528873443604,
+      "learning_rate": 4.0625000000000005e-06,
+      "loss": 0.0127,
+      "reward": 1.1113789081573486,
+      "reward_std": 3.429651975631714,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.0136209726333618,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 130
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 26.2,
+      "grad_norm": 0.6327362656593323,
+      "kl": 0.40226221084594727,
+      "learning_rate": 4.09375e-06,
+      "loss": 0.0161,
+      "reward": 0.7944153547286987,
+      "reward_std": 2.8826069831848145,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7250000238418579,
+      "rewards/wrapped_driving_reward": -1.3055846691131592,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 131
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 26.4,
+      "grad_norm": 0.6797881722450256,
+      "kl": 0.4582635164260864,
+      "learning_rate": 4.125e-06,
+      "loss": 0.0183,
+      "reward": 2.8031327724456787,
+      "reward_std": 0.7006269097328186,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.05313277989625931,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 132
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 26.6,
+      "grad_norm": 0.5752917528152466,
+      "kl": 0.36153456568717957,
+      "learning_rate": 4.15625e-06,
+      "loss": 0.0145,
+      "reward": -2.564105987548828,
+      "reward_std": 2.871788263320923,
+      "rewards/mpc_param_extraction_reward": 0.25,
+      "rewards/mpc_param_name_reward": 0.25,
+      "rewards/wrapped_driving_reward": -3.189105987548828,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 133
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 26.8,
+      "grad_norm": 0.569823145866394,
+      "kl": 0.3600581884384155,
+      "learning_rate": 4.1875e-06,
+      "loss": 0.0144,
+      "reward": 3.2037861347198486,
+      "reward_std": 0.1732039451599121,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8287861943244934,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 134
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 27.0,
+      "grad_norm": 11.942618370056152,
+      "kl": 2.177290678024292,
+      "learning_rate": 4.21875e-06,
+      "loss": 0.0871,
+      "reward": -2.3714582920074463,
+      "reward_std": 1.8921570777893066,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -3.7464582920074463,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 135
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 27.2,
+      "grad_norm": 0.5660642385482788,
+      "kl": 0.2908819019794464,
+      "learning_rate": 4.25e-06,
+      "loss": 0.0116,
+      "reward": -0.6192033290863037,
+      "reward_std": 3.6331570148468018,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -2.1192033290863037,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 136
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 27.4,
+      "grad_norm": 1.4041975736618042,
+      "kl": 0.463067889213562,
+      "learning_rate": 4.28125e-06,
+      "loss": 0.0185,
+      "reward": -2.75,
+      "reward_std": 1.1902379989624023,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 137
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 27.6,
+      "grad_norm": 0.4801470637321472,
+      "kl": 0.2532914876937866,
+      "learning_rate": 4.312500000000001e-06,
+      "loss": 0.0101,
+      "reward": -2.304798126220703,
+      "reward_std": 3.3904037475585938,
+      "rewards/mpc_param_extraction_reward": 0.25,
+      "rewards/mpc_param_name_reward": 0.25,
+      "rewards/wrapped_driving_reward": -3.054798126220703,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 138
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 27.8,
+      "grad_norm": 0.6999854445457458,
+      "kl": 0.4938638210296631,
+      "learning_rate": 4.3437500000000006e-06,
+      "loss": 0.0198,
+      "reward": -1.9556548595428467,
+      "reward_std": 3.430131196975708,
+      "rewards/mpc_param_extraction_reward": 0.25,
+      "rewards/mpc_param_name_reward": 0.25,
+      "rewards/wrapped_driving_reward": -2.9556548595428467,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 139
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 28.0,
+      "grad_norm": 1.7622352838516235,
+      "kl": 0.32535520195961,
+      "learning_rate": 4.3750000000000005e-06,
+      "loss": 0.013,
+      "reward": 3.048956871032715,
+      "reward_std": 0.7497459053993225,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.42395687103271484,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 140
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 28.2,
+      "grad_norm": 1.0910435914993286,
+      "kl": 0.3166691064834595,
+      "learning_rate": 4.40625e-06,
+      "loss": 0.0127,
+      "reward": 2.1717541217803955,
+      "reward_std": 2.45133900642395,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.45324593782424927,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 141
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 28.4,
+      "grad_norm": 0.563035249710083,
+      "kl": 0.34334975481033325,
+      "learning_rate": 4.4375e-06,
+      "loss": 0.0137,
+      "reward": -0.30545544624328613,
+      "reward_std": 2.531362295150757,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.71875,
+      "rewards/wrapped_driving_reward": -2.149205446243286,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 142
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 28.6,
+      "grad_norm": 0.6513370871543884,
+      "kl": 0.2893451154232025,
+      "learning_rate": 4.46875e-06,
+      "loss": 0.0116,
+      "reward": -0.7655331492424011,
+      "reward_std": 3.818908214569092,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -2.265533208847046,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 143
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 28.8,
+      "grad_norm": 0.6747258305549622,
+      "kl": 0.4012701213359833,
+      "learning_rate": 4.5e-06,
+      "loss": 0.0161,
+      "reward": -1.631712794303894,
+      "reward_std": 2.7382092475891113,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -3.2567129135131836,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 144
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 29.0,
+      "grad_norm": 1.388415813446045,
+      "kl": 0.3030587136745453,
+      "learning_rate": 4.53125e-06,
+      "loss": 0.0121,
+      "reward": -0.3709021210670471,
+      "reward_std": 3.0999691486358643,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.6666666865348816,
+      "rewards/wrapped_driving_reward": -2.0375688076019287,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 145
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 29.2,
+      "grad_norm": 1.3835958242416382,
+      "kl": 0.5185285806655884,
+      "learning_rate": 4.5625e-06,
+      "loss": 0.0207,
+      "reward": -0.25881457328796387,
+      "reward_std": 2.5411531925201416,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9010416865348816,
+      "rewards/wrapped_driving_reward": -2.6598563194274902,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 146
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 29.4,
+      "grad_norm": 1.0529229640960693,
+      "kl": 0.304034560918808,
+      "learning_rate": 4.59375e-06,
+      "loss": 0.0122,
+      "reward": 0.5033270120620728,
+      "reward_std": 3.8062596321105957,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.6216729879379272,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 147
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 29.6,
+      "grad_norm": 1.3924496173858643,
+      "kl": 0.3519279956817627,
+      "learning_rate": 4.625000000000001e-06,
+      "loss": 0.0141,
+      "reward": 1.081155776977539,
+      "reward_std": 2.07023286819458,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9750000238418579,
+      "rewards/wrapped_driving_reward": -1.1438441276550293,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 148
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 29.8,
+      "grad_norm": 1.6642379760742188,
+      "kl": 0.5595217347145081,
+      "learning_rate": 4.6562500000000005e-06,
+      "loss": 0.0224,
+      "reward": 2.879631519317627,
+      "reward_std": 0.5703426003456116,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.12963154911994934,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 149
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 30.0,
+      "grad_norm": 0.5775982737541199,
+      "kl": 0.2810514271259308,
+      "learning_rate": 4.6875000000000004e-06,
+      "loss": 0.0112,
+      "reward": 0.10444420576095581,
+      "reward_std": 3.2514774799346924,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -2.0205557346343994,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 150
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 30.2,
+      "grad_norm": 0.9198185801506042,
+      "kl": 0.28956174850463867,
+      "learning_rate": 4.71875e-06,
+      "loss": 0.0116,
+      "reward": -0.260436087846756,
+      "reward_std": 2.8927173614501953,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -2.2604360580444336,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 151
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 30.4,
+      "grad_norm": 0.7754166722297668,
+      "kl": 0.38463443517684937,
+      "learning_rate": 4.75e-06,
+      "loss": 0.0154,
+      "reward": 0.4493406414985657,
+      "reward_std": 2.646808385848999,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.300659418106079,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 152
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 30.6,
+      "grad_norm": 0.5780096650123596,
+      "kl": 0.3385607898235321,
+      "learning_rate": 4.781250000000001e-06,
+      "loss": 0.0135,
+      "reward": -1.7927734851837158,
+      "reward_std": 3.755190849304199,
+      "rewards/mpc_param_extraction_reward": 0.25,
+      "rewards/mpc_param_name_reward": 0.25,
+      "rewards/wrapped_driving_reward": -2.792773485183716,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 153
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 30.8,
+      "grad_norm": 0.5552729964256287,
+      "kl": 0.28436288237571716,
+      "learning_rate": 4.8125e-06,
+      "loss": 0.0114,
+      "reward": 0.6222386360168457,
+      "reward_std": 2.1850173473358154,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.8777613639831543,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 154
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 31.0,
+      "grad_norm": 0.9199939370155334,
+      "kl": 0.37593454122543335,
+      "learning_rate": 4.84375e-06,
+      "loss": 0.015,
+      "reward": 0.4306233525276184,
+      "reward_std": 3.5992963314056396,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.6607142686843872,
+      "rewards/wrapped_driving_reward": -1.605090856552124,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 155
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 31.2,
+      "grad_norm": 0.5603945851325989,
+      "kl": 0.3141997754573822,
+      "learning_rate": 4.875e-06,
+      "loss": 0.0126,
+      "reward": -2.0,
+      "reward_std": 1.3540064096450806,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 156
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 31.4,
+      "grad_norm": 0.6190344095230103,
+      "kl": 0.27537742257118225,
+      "learning_rate": 4.90625e-06,
+      "loss": 0.011,
+      "reward": 1.0282058715820312,
+      "reward_std": 3.394833564758301,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.846794068813324,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 157
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 31.6,
+      "grad_norm": 0.6877399682998657,
+      "kl": 0.2958383858203888,
+      "learning_rate": 4.937500000000001e-06,
+      "loss": 0.0118,
+      "reward": -0.755041241645813,
+      "reward_std": 3.4646472930908203,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -2.1300413608551025,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 158
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 31.8,
+      "grad_norm": 31.595932006835938,
+      "kl": 6.78364372253418,
+      "learning_rate": 4.9687500000000005e-06,
+      "loss": 0.2713,
+      "reward": 0.35857605934143066,
+      "reward_std": 2.9158554077148438,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.7664239406585693,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 159
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 32.0,
+      "grad_norm": 1.7048529386520386,
+      "kl": 0.35252463817596436,
+      "learning_rate": 5e-06,
+      "loss": 0.0141,
+      "reward": 2.64233136177063,
+      "reward_std": 0.7985239624977112,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.14233140647411346,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 160
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 32.2,
+      "grad_norm": 0.9699507355690002,
+      "kl": 0.3963090479373932,
+      "learning_rate": 4.99999405044338e-06,
+      "loss": 0.0159,
+      "reward": 2.578547477722168,
+      "reward_std": 0.16937123239040375,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.046452634036540985,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 161
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 32.4,
+      "grad_norm": 0.6427643299102783,
+      "kl": 0.2770542800426483,
+      "learning_rate": 4.999976201801837e-06,
+      "loss": 0.0111,
+      "reward": 2.2058539390563965,
+      "reward_std": 1.1022424697875977,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.41914597153663635,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 162
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 32.6,
+      "grad_norm": 0.6896190047264099,
+      "kl": 0.27440541982650757,
+      "learning_rate": 4.999946454160323e-06,
+      "loss": 0.011,
+      "reward": 1.1675429344177246,
+      "reward_std": 3.4587650299072266,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7142857313156128,
+      "rewards/wrapped_driving_reward": -0.7967426180839539,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 163
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 32.8,
+      "grad_norm": 0.7695831060409546,
+      "kl": 0.4198772609233856,
+      "learning_rate": 4.9999048076604286e-06,
+      "loss": 0.0168,
+      "reward": -1.912316918373108,
+      "reward_std": 2.555265426635742,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -2.9123167991638184,
+      "rewards/wrapped_format_reward": 0.0,
+      "step": 164
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 33.0,
+      "grad_norm": 0.5920954942703247,
+      "kl": 0.28969520330429077,
+      "learning_rate": 4.999851262500375e-06,
+      "loss": 0.0116,
+      "reward": 3.1377110481262207,
+      "reward_std": 0.5497580170631409,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.5127109289169312,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 165
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 33.2,
+      "grad_norm": 6.077564716339111,
+      "kl": 0.44747114181518555,
+      "learning_rate": 4.999785818935018e-06,
+      "loss": 0.0179,
+      "reward": 2.047877073287964,
+      "reward_std": 2.722182035446167,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.45212286710739136,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 166
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 33.4,
+      "grad_norm": 0.6889002919197083,
+      "kl": 0.37658053636550903,
+      "learning_rate": 4.999708477275846e-06,
+      "loss": 0.0151,
+      "reward": -2.284590482711792,
+      "reward_std": 3.106440782546997,
+      "rewards/mpc_param_extraction_reward": 0.25,
+      "rewards/mpc_param_name_reward": 0.25,
+      "rewards/wrapped_driving_reward": -3.159590482711792,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 167
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 33.6,
+      "grad_norm": 1.8645473718643188,
+      "kl": 0.3408987522125244,
+      "learning_rate": 4.9996192378909785e-06,
+      "loss": 0.0136,
+      "reward": 0.917718231678009,
+      "reward_std": 2.948974132537842,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.699999988079071,
+      "rewards/wrapped_driving_reward": -1.157281756401062,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 168
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 33.8,
+      "grad_norm": 0.535763680934906,
+      "kl": 0.25453072786331177,
+      "learning_rate": 4.999518101205162e-06,
+      "loss": 0.0102,
+      "reward": 3.604552745819092,
+      "reward_std": 0.45598289370536804,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.7295528054237366,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 169
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 34.0,
+      "grad_norm": 1.0853776931762695,
+      "kl": 0.2871979773044586,
+      "learning_rate": 4.999405067699773e-06,
+      "loss": 0.0115,
+      "reward": 0.7697337865829468,
+      "reward_std": 3.0176069736480713,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.3552662134170532,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 170
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 34.2,
+      "grad_norm": 2.175551176071167,
+      "kl": 0.7303879261016846,
+      "learning_rate": 4.99928013791281e-06,
+      "loss": 0.0292,
+      "reward": 0.010015249252319336,
+      "reward_std": 4.346557140350342,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.46875,
+      "rewards/wrapped_driving_reward": -1.5837347507476807,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 171
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 34.4,
+      "grad_norm": 1.3378883600234985,
+      "kl": 0.2555471658706665,
+      "learning_rate": 4.999143312438893e-06,
+      "loss": 0.0102,
+      "reward": 1.064118504524231,
+      "reward_std": 1.464298963546753,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.4358813762664795,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 172
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 34.6,
+      "grad_norm": 1.6005758047103882,
+      "kl": 0.3272940516471863,
+      "learning_rate": 4.998994591929266e-06,
+      "loss": 0.0131,
+      "reward": 3.320277214050293,
+      "reward_std": 0.5942137241363525,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8202772736549377,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 173
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 34.8,
+      "grad_norm": 0.8775622844696045,
+      "kl": 0.3981474041938782,
+      "learning_rate": 4.998833977091783e-06,
+      "loss": 0.0159,
+      "reward": 2.548191547393799,
+      "reward_std": 0.13038182258605957,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.17319151759147644,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 174
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 35.0,
+      "grad_norm": 0.5131356716156006,
+      "kl": 0.26495081186294556,
+      "learning_rate": 4.998661468690914e-06,
+      "loss": 0.0106,
+      "reward": 0.2881455421447754,
+      "reward_std": 3.1594552993774414,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.7118544578552246,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 175
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 35.2,
+      "grad_norm": 1.4990577697753906,
+      "kl": 0.3656232953071594,
+      "learning_rate": 4.99847706754774e-06,
+      "loss": 0.0146,
+      "reward": 2.0933961868286133,
+      "reward_std": 0.39702948927879333,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9375,
+      "rewards/wrapped_driving_reward": -0.594103991985321,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 176
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 35.4,
+      "grad_norm": 0.5740483999252319,
+      "kl": 0.265653520822525,
+      "learning_rate": 4.998280774539943e-06,
+      "loss": 0.0106,
+      "reward": 1.1700050830841064,
+      "reward_std": 3.1657402515411377,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.8299949765205383,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 177
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 35.6,
+      "grad_norm": 0.6564896702766418,
+      "kl": 0.265337198972702,
+      "learning_rate": 4.998072590601808e-06,
+      "loss": 0.0106,
+      "reward": -0.852949857711792,
+      "reward_std": 3.3822429180145264,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -2.102949857711792,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 178
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 35.8,
+      "grad_norm": 23.83641242980957,
+      "kl": 4.303451061248779,
+      "learning_rate": 4.9978525167242176e-06,
+      "loss": 0.1721,
+      "reward": 0.764412522315979,
+      "reward_std": 2.8684115409851074,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7250000238418579,
+      "rewards/wrapped_driving_reward": -1.085587501525879,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 179
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 36.0,
+      "grad_norm": 0.7350974082946777,
+      "kl": 0.30466321110725403,
+      "learning_rate": 4.997620553954645e-06,
+      "loss": 0.0122,
+      "reward": -0.10997164249420166,
+      "reward_std": 2.883012056350708,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -2.109971523284912,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 180
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 36.2,
+      "grad_norm": 1.8978265523910522,
+      "kl": 0.5050737857818604,
+      "learning_rate": 4.997376703397151e-06,
+      "loss": 0.0202,
+      "reward": -0.35431569814682007,
+      "reward_std": 4.209678649902344,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.8543156385421753,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 181
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 36.4,
+      "grad_norm": 0.6739000678062439,
+      "kl": 0.3342580497264862,
+      "learning_rate": 4.9971209662123774e-06,
+      "loss": 0.0134,
+      "reward": 1.24358332157135,
+      "reward_std": 3.5022475719451904,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.5064166188240051,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 182
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 36.6,
+      "grad_norm": 0.8527255654335022,
+      "kl": 0.44380900263786316,
+      "learning_rate": 4.996853343617542e-06,
+      "loss": 0.0178,
+      "reward": 1.3519909381866455,
+      "reward_std": 2.9203834533691406,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.71875,
+      "rewards/wrapped_driving_reward": -0.6167589426040649,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 183
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 36.8,
+      "grad_norm": 0.6037353277206421,
+      "kl": 0.3514931797981262,
+      "learning_rate": 4.9965738368864345e-06,
+      "loss": 0.0141,
+      "reward": 2.4617958068847656,
+      "reward_std": 0.43256813287734985,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.1632043421268463,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 184
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 37.0,
+      "grad_norm": 0.6498645544052124,
+      "kl": 0.39014145731925964,
+      "learning_rate": 4.996282447349408e-06,
+      "loss": 0.0156,
+      "reward": 2.696049451828003,
+      "reward_std": 0.6518055200576782,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.17895053327083588,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 185
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 37.2,
+      "grad_norm": 0.6228243708610535,
+      "kl": 0.2633248567581177,
+      "learning_rate": 4.995979176393372e-06,
+      "loss": 0.0105,
+      "reward": 1.1363269090652466,
+      "reward_std": 3.4644434452056885,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9886730313301086,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 186
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 37.4,
+      "grad_norm": 8.40079402923584,
+      "kl": 1.8278297185897827,
+      "learning_rate": 4.99566402546179e-06,
+      "loss": 0.0731,
+      "reward": -0.7244951725006104,
+      "reward_std": 3.783473491668701,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.3705357313156128,
+      "rewards/wrapped_driving_reward": -2.0950307846069336,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 187
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 37.6,
+      "grad_norm": 0.5168763399124146,
+      "kl": 0.2395801991224289,
+      "learning_rate": 4.995336996054668e-06,
+      "loss": 0.0096,
+      "reward": 1.9002426862716675,
+      "reward_std": 2.223823070526123,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9772727489471436,
+      "rewards/wrapped_driving_reward": -0.7020300626754761,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 188
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 37.8,
+      "grad_norm": 0.9863908290863037,
+      "kl": 0.27976277470588684,
+      "learning_rate": 4.99499808972855e-06,
+      "loss": 0.0112,
+      "reward": -0.028857052326202393,
+      "reward_std": 2.8702406883239746,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -2.1538569927215576,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 189
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 38.0,
+      "grad_norm": 0.8377166986465454,
+      "kl": 0.48623228073120117,
+      "learning_rate": 4.994647308096509e-06,
+      "loss": 0.0194,
+      "reward": 2.531177043914795,
+      "reward_std": 0.5673744082450867,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9166666865348816,
+      "rewards/wrapped_driving_reward": 0.4895104467868805,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 190
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 38.2,
+      "grad_norm": 0.9249876737594604,
+      "kl": 0.4526787996292114,
+      "learning_rate": 4.994284652828143e-06,
+      "loss": 0.0181,
+      "reward": 0.6909130215644836,
+      "reward_std": 3.1517491340637207,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.1840870380401611,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 191
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 38.4,
+      "grad_norm": 0.5216014385223389,
+      "kl": 0.2844958007335663,
+      "learning_rate": 4.993910125649561e-06,
+      "loss": 0.0114,
+      "reward": 1.347219705581665,
+      "reward_std": 3.583749771118164,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.527780294418335,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 192
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 38.6,
+      "grad_norm": 0.7675309181213379,
+      "kl": 0.46290096640586853,
+      "learning_rate": 4.99352372834338e-06,
+      "loss": 0.0185,
+      "reward": 1.28756582736969,
+      "reward_std": 3.200143337249756,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7083333134651184,
+      "rewards/wrapped_driving_reward": -0.5457674860954285,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 193
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 38.8,
+      "grad_norm": 0.5267873406410217,
+      "kl": 0.27246928215026855,
+      "learning_rate": 4.993125462748714e-06,
+      "loss": 0.0109,
+      "reward": 0.5119737386703491,
+      "reward_std": 2.572335958480835,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.9880262613296509,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 194
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 39.0,
+      "grad_norm": 0.557345449924469,
+      "kl": 0.33223679661750793,
+      "learning_rate": 4.992715330761167e-06,
+      "loss": 0.0133,
+      "reward": 1.9005041122436523,
+      "reward_std": 1.5405527353286743,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.5994958281517029,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 195
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 39.2,
+      "grad_norm": 0.5145586729049683,
+      "kl": 0.27872464060783386,
+      "learning_rate": 4.992293334332821e-06,
+      "loss": 0.0111,
+      "reward": 0.08070141077041626,
+      "reward_std": 2.161402702331543,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -2.9192986488342285,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 196
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 39.4,
+      "grad_norm": 0.5731538534164429,
+      "kl": 0.2947344481945038,
+      "learning_rate": 4.9918594754722286e-06,
+      "loss": 0.0118,
+      "reward": 1.089212417602539,
+      "reward_std": 3.5704760551452637,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9107875823974609,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 197
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 39.6,
+      "grad_norm": 1.0262069702148438,
+      "kl": 0.36793074011802673,
+      "learning_rate": 4.991413756244404e-06,
+      "loss": 0.0147,
+      "reward": 2.804293632507324,
+      "reward_std": 0.05172164365649223,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.3042936325073242,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 198
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 39.8,
+      "grad_norm": 0.7235340476036072,
+      "kl": 0.4867457151412964,
+      "learning_rate": 4.990956178770814e-06,
+      "loss": 0.0195,
+      "reward": 2.4924705028533936,
+      "reward_std": 0.6009870767593384,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.949999988079071,
+      "rewards/wrapped_driving_reward": -0.08252956718206406,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 199
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 40.0,
+      "grad_norm": 0.8564599752426147,
+      "kl": 0.4650922119617462,
+      "learning_rate": 4.990486745229364e-06,
+      "loss": 0.0186,
+      "reward": 2.757322311401367,
+      "reward_std": 0.5960695743560791,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9444444179534912,
+      "rewards/wrapped_driving_reward": 0.18787765502929688,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 200
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 40.2,
+      "grad_norm": 0.6181848645210266,
+      "kl": 0.33555763959884644,
+      "learning_rate": 4.990005457854392e-06,
+      "loss": 0.0134,
+      "reward": 0.935232937335968,
+      "reward_std": 2.9882521629333496,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7222222089767456,
+      "rewards/wrapped_driving_reward": -0.7869893312454224,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 201
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 40.4,
+      "grad_norm": 0.8061473369598389,
+      "kl": 0.3526011109352112,
+      "learning_rate": 4.989512318936654e-06,
+      "loss": 0.0141,
+      "reward": 2.038607597351074,
+      "reward_std": 1.286082148551941,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9772727489471436,
+      "rewards/wrapped_driving_reward": -0.4386652112007141,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 202
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 40.6,
+      "grad_norm": 1.0745853185653687,
+      "kl": 0.7225068807601929,
+      "learning_rate": 4.989007330823319e-06,
+      "loss": 0.0289,
+      "reward": 3.327683210372925,
+      "reward_std": 0.45302456617355347,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.5776832103729248,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 203
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 40.8,
+      "grad_norm": 0.6797990202903748,
+      "kl": 0.49457883834838867,
+      "learning_rate": 4.988490495917948e-06,
+      "loss": 0.0198,
+      "reward": 1.4564661979675293,
+      "reward_std": 3.6745243072509766,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.7935338020324707,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 204
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 41.0,
+      "grad_norm": 0.5719887018203735,
+      "kl": 0.3025702238082886,
+      "learning_rate": 4.987961816680493e-06,
+      "loss": 0.0121,
+      "reward": 0.8813665509223938,
+      "reward_std": 3.3135292530059814,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.685606062412262,
+      "rewards/wrapped_driving_reward": -1.1792395114898682,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 205
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 41.2,
+      "grad_norm": 0.7324315905570984,
+      "kl": 0.387521356344223,
+      "learning_rate": 4.987421295627279e-06,
+      "loss": 0.0155,
+      "reward": 3.60201358795166,
+      "reward_std": 0.17326904833316803,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9722222089767456,
+      "rewards/wrapped_driving_reward": 0.7547914981842041,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 206
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 41.4,
+      "grad_norm": 1.4426076412200928,
+      "kl": 0.3239262104034424,
+      "learning_rate": 4.986868935330998e-06,
+      "loss": 0.013,
+      "reward": 1.1451337337493896,
+      "reward_std": 3.175523042678833,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9798662662506104,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 207
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 41.6,
+      "grad_norm": 0.6265994310379028,
+      "kl": 0.31086966395378113,
+      "learning_rate": 4.986304738420684e-06,
+      "loss": 0.0124,
+      "reward": -0.08087223768234253,
+      "reward_std": 3.9480772018432617,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.5808722972869873,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 208
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 41.8,
+      "grad_norm": 0.5122293829917908,
+      "kl": 0.22147461771965027,
+      "learning_rate": 4.985728707581717e-06,
+      "loss": 0.0089,
+      "reward": 2.2255654335021973,
+      "reward_std": 0.4417201578617096,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9821428656578064,
+      "rewards/wrapped_driving_reward": -0.2565774619579315,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 209
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 42.0,
+      "grad_norm": 0.5366212725639343,
+      "kl": 0.2860429286956787,
+      "learning_rate": 4.985140845555799e-06,
+      "loss": 0.0114,
+      "reward": -1.875,
+      "reward_std": 1.108677864074707,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 210
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 42.2,
+      "grad_norm": 0.757074773311615,
+      "kl": 0.5041708946228027,
+      "learning_rate": 4.984541155140945e-06,
+      "loss": 0.0202,
+      "reward": 1.3050158023834229,
+      "reward_std": 3.2698206901550293,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.6428571343421936,
+      "rewards/wrapped_driving_reward": -0.7128414511680603,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 211
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 42.4,
+      "grad_norm": 0.5149911046028137,
+      "kl": 0.24131189286708832,
+      "learning_rate": 4.9839296391914696e-06,
+      "loss": 0.0097,
+      "reward": -0.5590072870254517,
+      "reward_std": 3.6906325817108154,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.9340074062347412,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 212
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 42.6,
+      "grad_norm": 0.7922428250312805,
+      "kl": 0.4074100852012634,
+      "learning_rate": 4.98330630061797e-06,
+      "loss": 0.0163,
+      "reward": 0.7251100540161133,
+      "reward_std": 3.205897569656372,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.1498900651931763,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 213
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 42.8,
+      "grad_norm": 0.8499237298965454,
+      "kl": 0.533706784248352,
+      "learning_rate": 4.982671142387316e-06,
+      "loss": 0.0213,
+      "reward": 1.2264912128448486,
+      "reward_std": 3.1925883293151855,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.7735086679458618,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 214
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 43.0,
+      "grad_norm": 0.5848891139030457,
+      "kl": 0.4833756983280182,
+      "learning_rate": 4.982024167522638e-06,
+      "loss": 0.0193,
+      "reward": 2.640871524810791,
+      "reward_std": 0.3350675404071808,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.935606062412262,
+      "rewards/wrapped_driving_reward": -0.1697344183921814,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 215
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 43.2,
+      "grad_norm": 1.0190398693084717,
+      "kl": 0.5212844014167786,
+      "learning_rate": 4.981365379103306e-06,
+      "loss": 0.0209,
+      "reward": 1.518845796585083,
+      "reward_std": 1.8981057405471802,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9772727489471436,
+      "rewards/wrapped_driving_reward": -1.0834269523620605,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 216
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 43.4,
+      "grad_norm": 0.6003134250640869,
+      "kl": 0.2476293295621872,
+      "learning_rate": 4.980694780264918e-06,
+      "loss": 0.0099,
+      "reward": 2.3462984561920166,
+      "reward_std": 0.5958766937255859,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.4037014842033386,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 217
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 43.6,
+      "grad_norm": 0.5352597832679749,
+      "kl": 0.33760789036750793,
+      "learning_rate": 4.980012374199288e-06,
+      "loss": 0.0135,
+      "reward": 1.1177078485488892,
+      "reward_std": 3.422083854675293,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.8822920918464661,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 218
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 43.8,
+      "grad_norm": 0.78425532579422,
+      "kl": 0.45192739367485046,
+      "learning_rate": 4.979318164154426e-06,
+      "loss": 0.0181,
+      "reward": 3.3331549167633057,
+      "reward_std": 0.4030221104621887,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8331548571586609,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 219
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 44.0,
+      "grad_norm": 0.5511319041252136,
+      "kl": 0.2625429630279541,
+      "learning_rate": 4.978612153434527e-06,
+      "loss": 0.0105,
+      "reward": 3.4739222526550293,
+      "reward_std": 0.35263335704803467,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.5989223122596741,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 220
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 44.2,
+      "grad_norm": 0.8345232009887695,
+      "kl": 0.5118071436882019,
+      "learning_rate": 4.97789434539995e-06,
+      "loss": 0.0205,
+      "reward": 1.788142442703247,
+      "reward_std": 2.3180289268493652,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.086857557296753,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 221
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 44.4,
+      "grad_norm": 0.8292976021766663,
+      "kl": 0.5234676003456116,
+      "learning_rate": 4.977164743467206e-06,
+      "loss": 0.0209,
+      "reward": 1.3859682083129883,
+      "reward_std": 3.6182608604431152,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.6875,
+      "rewards/wrapped_driving_reward": -0.5515317916870117,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 222
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 44.6,
+      "grad_norm": 0.8200549483299255,
+      "kl": 0.3950418531894684,
+      "learning_rate": 4.976423351108943e-06,
+      "loss": 0.0158,
+      "reward": 1.9203238487243652,
+      "reward_std": 1.1563453674316406,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.7046762704849243,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 223
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 44.8,
+      "grad_norm": 0.6968622207641602,
+      "kl": 0.2271728217601776,
+      "learning_rate": 4.975670171853926e-06,
+      "loss": 0.0091,
+      "reward": -0.3170052766799927,
+      "reward_std": 2.1093220710754395,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9772727489471436,
+      "rewards/wrapped_driving_reward": -2.919278144836426,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 224
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 45.0,
+      "grad_norm": 0.7795050144195557,
+      "kl": 0.4355601966381073,
+      "learning_rate": 4.97490520928702e-06,
+      "loss": 0.0174,
+      "reward": 2.6324033737182617,
+      "reward_std": 0.5314469933509827,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.11759641766548157,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 225
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 45.2,
+      "grad_norm": 0.5524005889892578,
+      "kl": 0.30952146649360657,
+      "learning_rate": 4.974128467049177e-06,
+      "loss": 0.0124,
+      "reward": -2.125,
+      "reward_std": 1.314977765083313,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 226
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 45.4,
+      "grad_norm": 0.5645884871482849,
+      "kl": 0.4939887821674347,
+      "learning_rate": 4.9733399488374115e-06,
+      "loss": 0.0198,
+      "reward": 2.418989658355713,
+      "reward_std": 0.14345024526119232,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.08101026713848114,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 227
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 45.6,
+      "grad_norm": 0.9631263017654419,
+      "kl": 0.6647568941116333,
+      "learning_rate": 4.972539658404793e-06,
+      "loss": 0.0266,
+      "reward": -0.0228692889213562,
+      "reward_std": 3.135000228881836,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -2.272869110107422,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 228
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 45.8,
+      "grad_norm": 0.5868902802467346,
+      "kl": 0.536701500415802,
+      "learning_rate": 4.971727599560418e-06,
+      "loss": 0.0215,
+      "reward": 2.595135450363159,
+      "reward_std": 0.5522119402885437,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.949999988079071,
+      "rewards/wrapped_driving_reward": 0.020135482773184776,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 229
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 46.0,
+      "grad_norm": 0.6927148103713989,
+      "kl": 0.5391973257064819,
+      "learning_rate": 4.970903776169403e-06,
+      "loss": 0.0216,
+      "reward": 3.2273426055908203,
+      "reward_std": 0.38745206594467163,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9642857313156128,
+      "rewards/wrapped_driving_reward": 0.7630569934844971,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 230
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 46.2,
+      "grad_norm": 2.157358407974243,
+      "kl": 0.5963761210441589,
+      "learning_rate": 4.9700681921528495e-06,
+      "loss": 0.0239,
+      "reward": 3.3556950092315674,
+      "reward_std": 0.5486971735954285,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.7306950092315674,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 231
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 46.4,
+      "grad_norm": 0.5409197211265564,
+      "kl": 0.31054040789604187,
+      "learning_rate": 4.9692208514878445e-06,
+      "loss": 0.0124,
+      "reward": -1.75,
+      "reward_std": 1.1902379989624023,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 232
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 46.6,
+      "grad_norm": 0.8271388411521912,
+      "kl": 0.5030784606933594,
+      "learning_rate": 4.968361758207428e-06,
+      "loss": 0.0201,
+      "reward": 2.2951016426086426,
+      "reward_std": 0.6324443817138672,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.07989836484193802,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 233
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 46.8,
+      "grad_norm": 0.9013113975524902,
+      "kl": 0.527148425579071,
+      "learning_rate": 4.9674909164005805e-06,
+      "loss": 0.0211,
+      "reward": -0.08311975002288818,
+      "reward_std": 4.243640422821045,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.5831197500228882,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 234
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 47.0,
+      "grad_norm": 0.621760368347168,
+      "kl": 0.5894174575805664,
+      "learning_rate": 4.966608330212198e-06,
+      "loss": 0.0236,
+      "reward": 2.69521427154541,
+      "reward_std": 0.2680894732475281,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9722222089767456,
+      "rewards/wrapped_driving_reward": -0.15200814604759216,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 235
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 47.2,
+      "grad_norm": 0.6673128604888916,
+      "kl": 0.42412999272346497,
+      "learning_rate": 4.965714003843079e-06,
+      "loss": 0.017,
+      "reward": -2.0,
+      "reward_std": 1.0801234245300293,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 236
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 47.4,
+      "grad_norm": 0.6826753616333008,
+      "kl": 0.48437440395355225,
+      "learning_rate": 4.9648079415499e-06,
+      "loss": 0.0194,
+      "reward": 2.6671550273895264,
+      "reward_std": 0.6421502828598022,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.20784501731395721,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 237
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 47.6,
+      "grad_norm": 0.7442097663879395,
+      "kl": 0.5179538130760193,
+      "learning_rate": 4.963890147645195e-06,
+      "loss": 0.0207,
+      "reward": 0.023519575595855713,
+      "reward_std": 1.7913424968719482,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -2.351480484008789,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 238
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 47.8,
+      "grad_norm": 0.9971833825111389,
+      "kl": 0.2566893994808197,
+      "learning_rate": 4.962960626497339e-06,
+      "loss": 0.0103,
+      "reward": 1.0741076469421387,
+      "reward_std": 3.4465811252593994,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.6758923530578613,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 239
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 48.0,
+      "grad_norm": 0.776371955871582,
+      "kl": 0.6667019724845886,
+      "learning_rate": 4.962019382530521e-06,
+      "loss": 0.0267,
+      "reward": 0.7681245803833008,
+      "reward_std": 3.63140606880188,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.6068754196166992,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 240
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 48.2,
+      "grad_norm": 0.958461344242096,
+      "kl": 0.6015651226043701,
+      "learning_rate": 4.961066420224729e-06,
+      "loss": 0.0241,
+      "reward": 0.8900174498558044,
+      "reward_std": 2.1547889709472656,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9583333134651184,
+      "rewards/wrapped_driving_reward": -1.568315863609314,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 241
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 48.4,
+      "grad_norm": 0.8577614426612854,
+      "kl": 0.7052382230758667,
+      "learning_rate": 4.960101744115727e-06,
+      "loss": 0.0282,
+      "reward": 0.500007152557373,
+      "reward_std": 3.6019463539123535,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.685606062412262,
+      "rewards/wrapped_driving_reward": -1.6855988502502441,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 242
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 48.6,
+      "grad_norm": 0.6088186502456665,
+      "kl": 0.3410260081291199,
+      "learning_rate": 4.959125358795031e-06,
+      "loss": 0.0136,
+      "reward": 1.2359226942062378,
+      "reward_std": 3.157292127609253,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.1390773057937622,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 243
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 48.8,
+      "grad_norm": 0.6780346035957336,
+      "kl": 0.47339513897895813,
+      "learning_rate": 4.958137268909887e-06,
+      "loss": 0.0189,
+      "reward": 1.3727295398712158,
+      "reward_std": 3.2822999954223633,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7250000238418579,
+      "rewards/wrapped_driving_reward": -0.8522703647613525,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 244
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 49.0,
+      "grad_norm": 0.6219626069068909,
+      "kl": 0.3212871849536896,
+      "learning_rate": 4.957137479163253e-06,
+      "loss": 0.0129,
+      "reward": 0.08353948593139648,
+      "reward_std": 2.884551525115967,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -2.2914605140686035,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 245
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 49.2,
+      "grad_norm": 0.8742188811302185,
+      "kl": 0.6009516716003418,
+      "learning_rate": 4.956125994313775e-06,
+      "loss": 0.024,
+      "reward": 3.219036817550659,
+      "reward_std": 0.6377858519554138,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8035714626312256,
+      "rewards/wrapped_driving_reward": 0.5404652953147888,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 246
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 49.4,
+      "grad_norm": 3.8272242546081543,
+      "kl": 1.5439887046813965,
+      "learning_rate": 4.95510281917576e-06,
+      "loss": 0.0618,
+      "reward": 3.679497241973877,
+      "reward_std": 0.29719072580337524,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.6794970631599426,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 247
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 49.6,
+      "grad_norm": 0.5449193120002747,
+      "kl": 0.3074452579021454,
+      "learning_rate": 4.9540679586191605e-06,
+      "loss": 0.0123,
+      "reward": -0.8099073171615601,
+      "reward_std": 2.768624782562256,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -2.9349074363708496,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 248
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 49.8,
+      "grad_norm": 0.9541939496994019,
+      "kl": 0.5022038221359253,
+      "learning_rate": 4.953021417569545e-06,
+      "loss": 0.0201,
+      "reward": 0.9676476120948792,
+      "reward_std": 3.3370866775512695,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.1573524475097656,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 249
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 50.0,
+      "grad_norm": 0.6913716197013855,
+      "kl": 0.22377586364746094,
+      "learning_rate": 4.9519632010080765e-06,
+      "loss": 0.009,
+      "reward": -0.7356908917427063,
+      "reward_std": 3.197190761566162,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -2.4856908321380615,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 250
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 50.2,
+      "grad_norm": 0.8929083347320557,
+      "kl": 0.5696563720703125,
+      "learning_rate": 4.950893313971492e-06,
+      "loss": 0.0228,
+      "reward": 1.0284161567687988,
+      "reward_std": 2.463923454284668,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.7215839624404907,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 251
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 50.4,
+      "grad_norm": 1.311846375465393,
+      "kl": 0.7255131006240845,
+      "learning_rate": 4.949811761552074e-06,
+      "loss": 0.029,
+      "reward": 1.2360342741012573,
+      "reward_std": 3.5064666271209717,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.6428571343421936,
+      "rewards/wrapped_driving_reward": -0.656822919845581,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 252
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 50.6,
+      "grad_norm": 0.4827212989330292,
+      "kl": 0.33761507272720337,
+      "learning_rate": 4.9487185488976284e-06,
+      "loss": 0.0135,
+      "reward": 1.2470874786376953,
+      "reward_std": 3.3136324882507324,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.8779124021530151,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 253
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 50.8,
+      "grad_norm": 0.846889078617096,
+      "kl": 0.7514812350273132,
+      "learning_rate": 4.94761368121146e-06,
+      "loss": 0.0301,
+      "reward": 2.474397897720337,
+      "reward_std": 0.5729619264602661,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.96875,
+      "rewards/wrapped_driving_reward": -0.24435219168663025,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 254
+    },
+    {
+      "completion_length": 467.0,
+      "epoch": 51.0,
+      "grad_norm": 0.6226330399513245,
+      "kl": 0.9035637378692627,
+      "learning_rate": 4.9464971637523465e-06,
+      "loss": 0.0361,
+      "reward": 2.976405620574951,
+      "reward_std": 0.597745954990387,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.023594465106725693,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 255
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 51.2,
+      "grad_norm": 0.6273528933525085,
+      "kl": 0.46313872933387756,
+      "learning_rate": 4.9453690018345144e-06,
+      "loss": 0.0185,
+      "reward": 1.4185447692871094,
+      "reward_std": 3.323413848876953,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.7064552307128906,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 256
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 51.4,
+      "grad_norm": 1.4358348846435547,
+      "kl": 0.3820998966693878,
+      "learning_rate": 4.944229200827616e-06,
+      "loss": 0.0153,
+      "reward": 0.3355594873428345,
+      "reward_std": 2.6716933250427246,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.414440393447876,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 257
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 51.6,
+      "grad_norm": 0.5715557932853699,
+      "kl": 0.26808997988700867,
+      "learning_rate": 4.943077766156698e-06,
+      "loss": 0.0107,
+      "reward": 1.8104004859924316,
+      "reward_std": 3.5490846633911133,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7272727489471436,
+      "rewards/wrapped_driving_reward": -0.4168723225593567,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 258
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 51.8,
+      "grad_norm": 0.9949742555618286,
+      "kl": 0.6875662803649902,
+      "learning_rate": 4.941914703302181e-06,
+      "loss": 0.0275,
+      "reward": 3.0740609169006348,
+      "reward_std": 1.2247024774551392,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": 0.5740607976913452,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 259
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 52.0,
+      "grad_norm": 0.5395461916923523,
+      "kl": 0.3153356909751892,
+      "learning_rate": 4.9407400177998335e-06,
+      "loss": 0.0126,
+      "reward": 1.252676010131836,
+      "reward_std": 3.17907452583313,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.6875,
+      "rewards/wrapped_driving_reward": -0.9348239898681641,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 260
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 52.2,
+      "grad_norm": 0.5175924301147461,
+      "kl": 0.41518455743789673,
+      "learning_rate": 4.939553715240741e-06,
+      "loss": 0.0166,
+      "reward": 3.3429102897644043,
+      "reward_std": 0.5592035055160522,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.46791017055511475,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 261
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 52.4,
+      "grad_norm": 0.47460633516311646,
+      "kl": 0.5516465306282043,
+      "learning_rate": 4.938355801271282e-06,
+      "loss": 0.0221,
+      "reward": 1.5110602378845215,
+      "reward_std": 1.9977182149887085,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9375,
+      "rewards/wrapped_driving_reward": -1.3014397621154785,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 262
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 52.6,
+      "grad_norm": 0.8124420046806335,
+      "kl": 0.5848581790924072,
+      "learning_rate": 4.937146281593103e-06,
+      "loss": 0.0234,
+      "reward": 3.247490406036377,
+      "reward_std": 0.6556951999664307,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.6224905252456665,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 263
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 52.8,
+      "grad_norm": 0.6221967339515686,
+      "kl": 0.30070972442626953,
+      "learning_rate": 4.935925161963089e-06,
+      "loss": 0.012,
+      "reward": 0.7041885852813721,
+      "reward_std": 3.16145658493042,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.295811414718628,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 264
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 53.0,
+      "grad_norm": 0.8025345206260681,
+      "kl": 0.4004298448562622,
+      "learning_rate": 4.9346924481933345e-06,
+      "loss": 0.016,
+      "reward": -2.3282265663146973,
+      "reward_std": 1.6561260223388672,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -3.9532265663146973,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 265
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 53.2,
+      "grad_norm": 2.1500437259674072,
+      "kl": 0.9270884990692139,
+      "learning_rate": 4.933448146151122e-06,
+      "loss": 0.0371,
+      "reward": 3.697523355484009,
+      "reward_std": 0.2363002598285675,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8225233554840088,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 266
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 53.4,
+      "grad_norm": 0.9306778311729431,
+      "kl": 0.46557193994522095,
+      "learning_rate": 4.932192261758885e-06,
+      "loss": 0.0186,
+      "reward": 3.163017749786377,
+      "reward_std": 0.2954404056072235,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9772727489471436,
+      "rewards/wrapped_driving_reward": 0.3107450008392334,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 267
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 53.6,
+      "grad_norm": 0.7212801575660706,
+      "kl": 0.5435701608657837,
+      "learning_rate": 4.930924800994192e-06,
+      "loss": 0.0217,
+      "reward": 1.4288103580474854,
+      "reward_std": 3.6827523708343506,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.5711897015571594,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 268
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 53.8,
+      "grad_norm": 0.8867128491401672,
+      "kl": 0.7739095687866211,
+      "learning_rate": 4.929645769889704e-06,
+      "loss": 0.031,
+      "reward": -2.1571238040924072,
+      "reward_std": 1.6486133337020874,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -3.7821238040924072,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 269
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 54.0,
+      "grad_norm": 0.510140061378479,
+      "kl": 0.8436269164085388,
+      "learning_rate": 4.928355174533153e-06,
+      "loss": 0.0337,
+      "reward": 2.837273120880127,
+      "reward_std": 0.3524271845817566,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.03772694990038872,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 270
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 54.2,
+      "grad_norm": 0.6447189450263977,
+      "kl": 0.37782302498817444,
+      "learning_rate": 4.927053021067321e-06,
+      "loss": 0.0151,
+      "reward": 2.576737642288208,
+      "reward_std": 0.5483171939849854,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.326737642288208,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 271
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 54.4,
+      "grad_norm": 0.670859158039093,
+      "kl": 1.0185219049453735,
+      "learning_rate": 4.925739315689991e-06,
+      "loss": 0.0407,
+      "reward": 2.652383327484131,
+      "reward_std": 0.27769026160240173,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9166666865348816,
+      "rewards/wrapped_driving_reward": -0.1392831951379776,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 272
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 54.6,
+      "grad_norm": 0.5182738304138184,
+      "kl": 0.6035473942756653,
+      "learning_rate": 4.924414064653938e-06,
+      "loss": 0.0241,
+      "reward": -1.0863802433013916,
+      "reward_std": 2.398730754852295,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7272727489471436,
+      "rewards/wrapped_driving_reward": -3.188652992248535,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 273
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 54.8,
+      "grad_norm": 0.506093442440033,
+      "kl": 0.23574630916118622,
+      "learning_rate": 4.923077274266886e-06,
+      "loss": 0.0094,
+      "reward": 1.0250887870788574,
+      "reward_std": 2.6349871158599854,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.8499112129211426,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 274
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 55.0,
+      "grad_norm": 1.008044719696045,
+      "kl": 1.0019625425338745,
+      "learning_rate": 4.9217289508914836e-06,
+      "loss": 0.0401,
+      "reward": 3.194295883178711,
+      "reward_std": 0.7734468579292297,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.44429582357406616,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 275
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 55.2,
+      "grad_norm": 0.7316671013832092,
+      "kl": 0.44905000925064087,
+      "learning_rate": 4.92036910094527e-06,
+      "loss": 0.018,
+      "reward": 2.1943540573120117,
+      "reward_std": 1.172672152519226,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9750000238418579,
+      "rewards/wrapped_driving_reward": -0.40564602613449097,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 276
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 55.4,
+      "grad_norm": 0.5603216290473938,
+      "kl": 0.6424278616905212,
+      "learning_rate": 4.91899773090065e-06,
+      "loss": 0.0257,
+      "reward": 2.5801260471343994,
+      "reward_std": 0.4576241075992584,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.16987384855747223,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 277
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 55.6,
+      "grad_norm": 0.7360312938690186,
+      "kl": 0.760132372379303,
+      "learning_rate": 4.917614847284858e-06,
+      "loss": 0.0304,
+      "reward": 3.1857800483703613,
+      "reward_std": 0.6251810193061829,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.5607799887657166,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 278
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 55.8,
+      "grad_norm": 0.6797391176223755,
+      "kl": 0.7862927317619324,
+      "learning_rate": 4.91622045667993e-06,
+      "loss": 0.0315,
+      "reward": 1.7073078155517578,
+      "reward_std": 3.480921983718872,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.542692244052887,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 279
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 56.0,
+      "grad_norm": 0.7321208119392395,
+      "kl": 0.5703269243240356,
+      "learning_rate": 4.914814565722671e-06,
+      "loss": 0.0228,
+      "reward": 2.8495311737060547,
+      "reward_std": 0.2323673665523529,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9583333134651184,
+      "rewards/wrapped_driving_reward": 0.01619771495461464,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 280
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 56.2,
+      "grad_norm": 0.5318320989608765,
+      "kl": 0.695686936378479,
+      "learning_rate": 4.913397181104623e-06,
+      "loss": 0.0278,
+      "reward": 1.8786531686782837,
+      "reward_std": 3.592921257019043,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.3713468611240387,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 281
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 56.4,
+      "grad_norm": 1.4247713088989258,
+      "kl": 0.6277374029159546,
+      "learning_rate": 4.9119683095720325e-06,
+      "loss": 0.0251,
+      "reward": -1.9102458953857422,
+      "reward_std": 3.205610513687134,
+      "rewards/mpc_param_extraction_reward": 0.25,
+      "rewards/mpc_param_name_reward": 0.25,
+      "rewards/wrapped_driving_reward": -3.035245895385742,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 282
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 56.6,
+      "grad_norm": 0.5404212474822998,
+      "kl": 0.5326585173606873,
+      "learning_rate": 4.9105279579258234e-06,
+      "loss": 0.0213,
+      "reward": 2.9429216384887695,
+      "reward_std": 0.2231481820344925,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9772727489471436,
+      "rewards/wrapped_driving_reward": 0.0906490758061409,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 283
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 56.8,
+      "grad_norm": 0.649975061416626,
+      "kl": 0.692694365978241,
+      "learning_rate": 4.909076133021558e-06,
+      "loss": 0.0277,
+      "reward": 2.443101406097412,
+      "reward_std": 0.7186054587364197,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.06810133904218674,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 284
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 57.0,
+      "grad_norm": 1.1204376220703125,
+      "kl": 0.5154175162315369,
+      "learning_rate": 4.907612841769407e-06,
+      "loss": 0.0206,
+      "reward": 3.386246681213379,
+      "reward_std": 0.1496068835258484,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.7612467408180237,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 285
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 57.2,
+      "grad_norm": 0.734048068523407,
+      "kl": 0.5711463093757629,
+      "learning_rate": 4.906138091134118e-06,
+      "loss": 0.0228,
+      "reward": 1.2789300680160522,
+      "reward_std": 3.544008731842041,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9710699319839478,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 286
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 57.4,
+      "grad_norm": 0.5225769281387329,
+      "kl": 0.27049189805984497,
+      "learning_rate": 4.904651888134982e-06,
+      "loss": 0.0108,
+      "reward": 3.3136672973632812,
+      "reward_std": 0.08986721932888031,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9722222089767456,
+      "rewards/wrapped_driving_reward": 0.3414452075958252,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 287
+    },
+    {
+      "completion_length": 413.0,
+      "epoch": 57.6,
+      "grad_norm": 1.317987322807312,
+      "kl": 0.8254587650299072,
+      "learning_rate": 4.903154239845798e-06,
+      "loss": 0.033,
+      "reward": 3.501180648803711,
+      "reward_std": 0.3513033092021942,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.6261807084083557,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 288
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 57.8,
+      "grad_norm": 0.6399166584014893,
+      "kl": 0.6499161124229431,
+      "learning_rate": 4.901645153394838e-06,
+      "loss": 0.026,
+      "reward": 2.196173667907715,
+      "reward_std": 1.2169743776321411,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9545454382896423,
+      "rewards/wrapped_driving_reward": -0.38337159156799316,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 289
+    },
+    {
+      "completion_length": 465.0,
+      "epoch": 58.0,
+      "grad_norm": 0.57232266664505,
+      "kl": 0.6679652333259583,
+      "learning_rate": 4.900124635964823e-06,
+      "loss": 0.0267,
+      "reward": 2.7008635997772217,
+      "reward_std": 0.3644496500492096,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.17413626611232758,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 290
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 58.2,
+      "grad_norm": 0.5505650639533997,
+      "kl": 0.6699390411376953,
+      "learning_rate": 4.898592694792871e-06,
+      "loss": 0.0268,
+      "reward": 3.24006724357605,
+      "reward_std": 0.26251503825187683,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.7400672435760498,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 291
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 58.4,
+      "grad_norm": 0.6135743260383606,
+      "kl": 0.41203054785728455,
+      "learning_rate": 4.897049337170483e-06,
+      "loss": 0.0165,
+      "reward": 2.3971381187438965,
+      "reward_std": 2.2839035987854004,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.4778619110584259,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 292
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 58.6,
+      "grad_norm": 0.5831702351570129,
+      "kl": 0.4307749271392822,
+      "learning_rate": 4.895494570443492e-06,
+      "loss": 0.0172,
+      "reward": -0.7622057199478149,
+      "reward_std": 2.7372794151306152,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -3.0122056007385254,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 293
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 58.8,
+      "grad_norm": 0.5363391637802124,
+      "kl": 0.3049720525741577,
+      "learning_rate": 4.8939284020120365e-06,
+      "loss": 0.0122,
+      "reward": 1.0504395961761475,
+      "reward_std": 3.033752202987671,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.699999988079071,
+      "rewards/wrapped_driving_reward": -1.274560570716858,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 294
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 59.0,
+      "grad_norm": 0.6540634632110596,
+      "kl": 0.2309640794992447,
+      "learning_rate": 4.8923508393305224e-06,
+      "loss": 0.0092,
+      "reward": 1.0688008069992065,
+      "reward_std": 3.047051429748535,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9311991333961487,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 295
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 59.2,
+      "grad_norm": 0.7878542542457581,
+      "kl": 0.44138067960739136,
+      "learning_rate": 4.890761889907589e-06,
+      "loss": 0.0177,
+      "reward": 1.3176345825195312,
+      "reward_std": 2.8866841793060303,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.8073654770851135,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 296
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 59.4,
+      "grad_norm": 0.5553893446922302,
+      "kl": 0.6941342949867249,
+      "learning_rate": 4.8891615613060715e-06,
+      "loss": 0.0278,
+      "reward": 2.6625680923461914,
+      "reward_std": 0.46423208713531494,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.037567950785160065,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 297
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 59.6,
+      "grad_norm": 0.5995137095451355,
+      "kl": 0.9473585486412048,
+      "learning_rate": 4.887549861142967e-06,
+      "loss": 0.0379,
+      "reward": 1.7219200134277344,
+      "reward_std": 2.298279047012329,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.9030801057815552,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 298
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 59.8,
+      "grad_norm": 0.58851557970047,
+      "kl": 0.5266136527061462,
+      "learning_rate": 4.885926797089396e-06,
+      "loss": 0.0211,
+      "reward": 2.1079962253570557,
+      "reward_std": 0.7257985472679138,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.90625,
+      "rewards/wrapped_driving_reward": -0.2982538044452667,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 299
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 60.0,
+      "grad_norm": 0.5463822484016418,
+      "kl": 0.4176102578639984,
+      "learning_rate": 4.884292376870567e-06,
+      "loss": 0.0167,
+      "reward": 3.0649795532226562,
+      "reward_std": 0.5604602694511414,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.06497950851917267,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 300
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 60.2,
+      "grad_norm": 0.6283360123634338,
+      "kl": 0.7974268198013306,
+      "learning_rate": 4.882646608265743e-06,
+      "loss": 0.0319,
+      "reward": 2.354006290435791,
+      "reward_std": 2.2440249919891357,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.6459937691688538,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 301
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 60.4,
+      "grad_norm": 0.5363072156906128,
+      "kl": 0.3808169662952423,
+      "learning_rate": 4.880989499108196e-06,
+      "loss": 0.0152,
+      "reward": 2.4912989139556885,
+      "reward_std": 0.2686402499675751,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.2587011158466339,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 302
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 60.6,
+      "grad_norm": 0.7250503897666931,
+      "kl": 0.9471665620803833,
+      "learning_rate": 4.8793210572851795e-06,
+      "loss": 0.0379,
+      "reward": 1.964458703994751,
+      "reward_std": 2.0956764221191406,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.6605411767959595,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 303
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 60.8,
+      "grad_norm": 0.5111416578292847,
+      "kl": 0.3088001608848572,
+      "learning_rate": 4.8776412907378845e-06,
+      "loss": 0.0124,
+      "reward": -1.0382410287857056,
+      "reward_std": 2.2208359241485596,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -3.038240909576416,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 304
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 61.0,
+      "grad_norm": 0.5215743780136108,
+      "kl": 0.24940745532512665,
+      "learning_rate": 4.875950207461403e-06,
+      "loss": 0.01,
+      "reward": 1.1871674060821533,
+      "reward_std": 1.8741310834884644,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.1878325939178467,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 305
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 61.2,
+      "grad_norm": 0.6458624005317688,
+      "kl": 0.5968429446220398,
+      "learning_rate": 4.874247815504693e-06,
+      "loss": 0.0239,
+      "reward": 1.3467873334884644,
+      "reward_std": 3.565579414367676,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9032126665115356,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 306
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 61.4,
+      "grad_norm": 0.5787416696548462,
+      "kl": 0.3611050844192505,
+      "learning_rate": 4.872534122970536e-06,
+      "loss": 0.0144,
+      "reward": -1.5641289949417114,
+      "reward_std": 2.654411792755127,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -2.689128875732422,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 307
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 61.6,
+      "grad_norm": 0.9745285511016846,
+      "kl": 1.186964988708496,
+      "learning_rate": 4.870809138015499e-06,
+      "loss": 0.0475,
+      "reward": 3.2458059787750244,
+      "reward_std": 0.423534095287323,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.4958060383796692,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 308
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 61.8,
+      "grad_norm": 0.5002840757369995,
+      "kl": 0.35287949442863464,
+      "learning_rate": 4.8690728688499e-06,
+      "loss": 0.0141,
+      "reward": -0.37188810110092163,
+      "reward_std": 2.867581367492676,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.699999988079071,
+      "rewards/wrapped_driving_reward": -2.821887969970703,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 309
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 62.0,
+      "grad_norm": 0.5000089406967163,
+      "kl": 0.49758777022361755,
+      "learning_rate": 4.867325323737765e-06,
+      "loss": 0.0199,
+      "reward": 1.0681167840957642,
+      "reward_std": 3.100673198699951,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7222222089767456,
+      "rewards/wrapped_driving_reward": -1.2791054248809814,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 310
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 62.2,
+      "grad_norm": 0.5534230470657349,
+      "kl": 0.2521771788597107,
+      "learning_rate": 4.865566510996787e-06,
+      "loss": 0.0101,
+      "reward": 1.340180516242981,
+      "reward_std": 2.2682108879089355,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.2848193645477295,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 311
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 62.4,
+      "grad_norm": 0.8099197149276733,
+      "kl": 0.9075281023979187,
+      "learning_rate": 4.863796438998293e-06,
+      "loss": 0.0363,
+      "reward": 1.2343413829803467,
+      "reward_std": 3.5059289932250977,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7083333134651184,
+      "rewards/wrapped_driving_reward": -0.723991870880127,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 312
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 62.6,
+      "grad_norm": 0.5338905453681946,
+      "kl": 0.6918814778327942,
+      "learning_rate": 4.862015116167195e-06,
+      "loss": 0.0277,
+      "reward": -0.8979493975639343,
+      "reward_std": 2.7375407218933105,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -3.022949457168579,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 313
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 62.8,
+      "grad_norm": 0.4916922450065613,
+      "kl": 0.46378105878829956,
+      "learning_rate": 4.860222550981961e-06,
+      "loss": 0.0186,
+      "reward": 3.5812926292419434,
+      "reward_std": 0.48870983719825745,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.831292450428009,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 314
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 63.0,
+      "grad_norm": 0.5365267395973206,
+      "kl": 0.9468160271644592,
+      "learning_rate": 4.858418751974564e-06,
+      "loss": 0.0379,
+      "reward": 2.7630491256713867,
+      "reward_std": 0.2816019356250763,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.013049202039837837,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 315
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 63.2,
+      "grad_norm": 0.6400690674781799,
+      "kl": 0.4425306022167206,
+      "learning_rate": 4.856603727730446e-06,
+      "loss": 0.0177,
+      "reward": 0.8091722130775452,
+      "reward_std": 3.2286431789398193,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7361111044883728,
+      "rewards/wrapped_driving_reward": -1.176938772201538,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 316
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 63.4,
+      "grad_norm": 0.6683816909790039,
+      "kl": 0.5423528552055359,
+      "learning_rate": 4.854777486888481e-06,
+      "loss": 0.0217,
+      "reward": 1.423877239227295,
+      "reward_std": 2.06416654586792,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9318181872367859,
+      "rewards/wrapped_driving_reward": -1.0079410076141357,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 317
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 63.6,
+      "grad_norm": 0.5277029871940613,
+      "kl": 0.49146440625190735,
+      "learning_rate": 4.852940038140927e-06,
+      "loss": 0.0197,
+      "reward": 3.4070889949798584,
+      "reward_std": 0.4839623272418976,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.532089114189148,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 318
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 63.8,
+      "grad_norm": 0.8004822134971619,
+      "kl": 0.6069704294204712,
+      "learning_rate": 4.8510913902333876e-06,
+      "loss": 0.0243,
+      "reward": 1.4200356006622314,
+      "reward_std": 3.2922091484069824,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9549642205238342,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 319
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 64.0,
+      "grad_norm": 0.5641809105873108,
+      "kl": 0.47654300928115845,
+      "learning_rate": 4.849231551964771e-06,
+      "loss": 0.0191,
+      "reward": 1.7679204940795898,
+      "reward_std": 2.522942543029785,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.48207950592041016,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 320
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 64.2,
+      "grad_norm": 0.6091551184654236,
+      "kl": 0.25179192423820496,
+      "learning_rate": 4.8473605321872484e-06,
+      "loss": 0.0101,
+      "reward": -0.5306634306907654,
+      "reward_std": 2.778803825378418,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -2.65566349029541,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 321
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 64.4,
+      "grad_norm": 0.5598704218864441,
+      "kl": 0.5083995461463928,
+      "learning_rate": 4.845478339806211e-06,
+      "loss": 0.0203,
+      "reward": 1.3865933418273926,
+      "reward_std": 3.261355400085449,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.6439394354820251,
+      "rewards/wrapped_driving_reward": -0.3823460340499878,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 322
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 64.6,
+      "grad_norm": 1.2819907665252686,
+      "kl": 0.26251715421676636,
+      "learning_rate": 4.843584983780225e-06,
+      "loss": 0.0105,
+      "reward": 2.78169846534729,
+      "reward_std": 0.5642634034156799,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.53169846534729,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 323
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 64.8,
+      "grad_norm": 0.5119650363922119,
+      "kl": 0.44216257333755493,
+      "learning_rate": 4.841680473120994e-06,
+      "loss": 0.0177,
+      "reward": 1.7560722827911377,
+      "reward_std": 0.8860724568367004,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8999999761581421,
+      "rewards/wrapped_driving_reward": -0.893927812576294,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 324
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 65.0,
+      "grad_norm": 0.5220951437950134,
+      "kl": 0.3728632926940918,
+      "learning_rate": 4.839764816893315e-06,
+      "loss": 0.0149,
+      "reward": 0.11694353818893433,
+      "reward_std": 3.4642200469970703,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.6625000238418579,
+      "rewards/wrapped_driving_reward": -1.7955565452575684,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 325
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 65.2,
+      "grad_norm": 0.7010544538497925,
+      "kl": 0.6249963641166687,
+      "learning_rate": 4.83783802421503e-06,
+      "loss": 0.025,
+      "reward": 2.118408441543579,
+      "reward_std": 0.14386098086833954,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.3815915584564209,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 326
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 65.4,
+      "grad_norm": 0.6010672450065613,
+      "kl": 0.8014960289001465,
+      "learning_rate": 4.835900104256989e-06,
+      "loss": 0.0321,
+      "reward": 3.40175724029541,
+      "reward_std": 0.4777882695198059,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.7767573595046997,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 327
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 65.6,
+      "grad_norm": 0.7275899052619934,
+      "kl": 1.015032410621643,
+      "learning_rate": 4.833951066243004e-06,
+      "loss": 0.0406,
+      "reward": 1.5494968891143799,
+      "reward_std": 3.7214293479919434,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.625,
+      "rewards/wrapped_driving_reward": -0.5755031704902649,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 328
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 65.8,
+      "grad_norm": 0.512077271938324,
+      "kl": 0.4679301083087921,
+      "learning_rate": 4.831990919449806e-06,
+      "loss": 0.0187,
+      "reward": 1.843889832496643,
+      "reward_std": 1.9196797609329224,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9583333134651184,
+      "rewards/wrapped_driving_reward": -0.9894434213638306,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 329
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 66.0,
+      "grad_norm": 0.5223225355148315,
+      "kl": 0.5572952628135681,
+      "learning_rate": 4.830019673206997e-06,
+      "loss": 0.0223,
+      "reward": 1.724566102027893,
+      "reward_std": 0.6851178407669067,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.7754338979721069,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 330
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 66.2,
+      "grad_norm": 0.8261262774467468,
+      "kl": 0.6189790964126587,
+      "learning_rate": 4.828037336897009e-06,
+      "loss": 0.0248,
+      "reward": 2.4810099601745605,
+      "reward_std": 0.7533643245697021,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9444444179534912,
+      "rewards/wrapped_driving_reward": -0.21343453228473663,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 331
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 66.4,
+      "grad_norm": 0.7530797123908997,
+      "kl": 1.0583492517471313,
+      "learning_rate": 4.826043919955062e-06,
+      "loss": 0.0423,
+      "reward": 0.9729395508766174,
+      "reward_std": 1.709369421005249,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.5270603895187378,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 332
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 66.6,
+      "grad_norm": 0.7703694105148315,
+      "kl": 1.1116454601287842,
+      "learning_rate": 4.824039431869112e-06,
+      "loss": 0.0445,
+      "reward": 2.3814101219177246,
+      "reward_std": 0.2855786383152008,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.118589848279953,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 333
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 66.8,
+      "grad_norm": 1.2055093050003052,
+      "kl": 1.0002336502075195,
+      "learning_rate": 4.822023882179811e-06,
+      "loss": 0.04,
+      "reward": 1.3947391510009766,
+      "reward_std": 3.263190746307373,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9802609086036682,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 334
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 67.0,
+      "grad_norm": 0.5130785703659058,
+      "kl": 0.5356588959693909,
+      "learning_rate": 4.8199972804804615e-06,
+      "loss": 0.0214,
+      "reward": 1.5739116668701172,
+      "reward_std": 3.7419235706329346,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7142857313156128,
+      "rewards/wrapped_driving_reward": -0.3903741240501404,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 335
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 67.2,
+      "grad_norm": 0.5589498281478882,
+      "kl": 0.5461040139198303,
+      "learning_rate": 4.817959636416969e-06,
+      "loss": 0.0218,
+      "reward": 0.7681459784507751,
+      "reward_std": 2.159348726272583,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.949999988079071,
+      "rewards/wrapped_driving_reward": -1.931854009628296,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 336
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 67.4,
+      "grad_norm": 0.5277290344238281,
+      "kl": 0.3753708004951477,
+      "learning_rate": 4.815910959687795e-06,
+      "loss": 0.015,
+      "reward": 2.5270836353302,
+      "reward_std": 0.9810623526573181,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.22291645407676697,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 337
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 67.6,
+      "grad_norm": 0.6664614081382751,
+      "kl": 0.4376363158226013,
+      "learning_rate": 4.8138512600439165e-06,
+      "loss": 0.0175,
+      "reward": 0.6591283679008484,
+      "reward_std": 3.1547491550445557,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.5908715724945068,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 338
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 67.8,
+      "grad_norm": 0.6150086522102356,
+      "kl": 1.0432140827178955,
+      "learning_rate": 4.8117805472887706e-06,
+      "loss": 0.0417,
+      "reward": 1.9501566886901855,
+      "reward_std": 3.63374924659729,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7272727489471436,
+      "rewards/wrapped_driving_reward": -0.4021160304546356,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 339
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 68.0,
+      "grad_norm": 0.6247034072875977,
+      "kl": 0.5003357529640198,
+      "learning_rate": 4.809698831278217e-06,
+      "loss": 0.02,
+      "reward": 3.1344668865203857,
+      "reward_std": 0.4356966018676758,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.949999988079071,
+      "rewards/wrapped_driving_reward": 0.5594670176506042,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 340
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 68.2,
+      "grad_norm": 0.7365332245826721,
+      "kl": 0.9077324867248535,
+      "learning_rate": 4.807606121920486e-06,
+      "loss": 0.0363,
+      "reward": 2.7017669677734375,
+      "reward_std": 0.26797592639923096,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.1732332408428192,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 341
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 68.4,
+      "grad_norm": 0.5844131112098694,
+      "kl": 0.7788793444633484,
+      "learning_rate": 4.80550242917613e-06,
+      "loss": 0.0312,
+      "reward": 1.429833173751831,
+      "reward_std": 1.8360050916671753,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.930555522441864,
+      "rewards/wrapped_driving_reward": -1.2507224082946777,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 342
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 68.6,
+      "grad_norm": 0.5121277570724487,
+      "kl": 0.41953617334365845,
+      "learning_rate": 4.803387763057981e-06,
+      "loss": 0.0168,
+      "reward": 1.4760076999664307,
+      "reward_std": 3.3271522521972656,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.7739923000335693,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 343
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 68.8,
+      "grad_norm": 0.6013743877410889,
+      "kl": 0.8165791630744934,
+      "learning_rate": 4.801262133631101e-06,
+      "loss": 0.0327,
+      "reward": 1.8635355234146118,
+      "reward_std": 3.9091763496398926,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.3864644765853882,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 344
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 69.0,
+      "grad_norm": 0.6182828545570374,
+      "kl": 1.0937128067016602,
+      "learning_rate": 4.799125551012731e-06,
+      "loss": 0.0437,
+      "reward": 3.419711112976074,
+      "reward_std": 0.48218590021133423,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9375,
+      "rewards/wrapped_driving_reward": 0.7322112321853638,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 345
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 69.2,
+      "grad_norm": 0.6208542585372925,
+      "kl": 0.6823076605796814,
+      "learning_rate": 4.796978025372247e-06,
+      "loss": 0.0273,
+      "reward": 0.9823777079582214,
+      "reward_std": 2.9922034740448,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7250000238418579,
+      "rewards/wrapped_driving_reward": -0.9926222562789917,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 346
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 69.4,
+      "grad_norm": 0.46940603852272034,
+      "kl": 0.6684026122093201,
+      "learning_rate": 4.794819566931107e-06,
+      "loss": 0.0267,
+      "reward": 1.6367030143737793,
+      "reward_std": 3.091221809387207,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.4882969856262207,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 347
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 69.6,
+      "grad_norm": 0.6682089567184448,
+      "kl": 0.5266464352607727,
+      "learning_rate": 4.79265018596281e-06,
+      "loss": 0.0211,
+      "reward": 0.8233842253684998,
+      "reward_std": 2.3113174438476562,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.926615595817566,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 348
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 69.8,
+      "grad_norm": 0.49833944439888,
+      "kl": 0.22715015709400177,
+      "learning_rate": 4.79046989279284e-06,
+      "loss": 0.0091,
+      "reward": 0.9500528573989868,
+      "reward_std": 2.9911322593688965,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.1749471426010132,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 349
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 70.0,
+      "grad_norm": 0.5571979880332947,
+      "kl": 0.39211493730545044,
+      "learning_rate": 4.788278697798619e-06,
+      "loss": 0.0157,
+      "reward": 3.3137106895446777,
+      "reward_std": 0.5589663982391357,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8137108087539673,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 350
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 70.2,
+      "grad_norm": 0.6056340932846069,
+      "kl": 0.5785823464393616,
+      "learning_rate": 4.7860766114094555e-06,
+      "loss": 0.0231,
+      "reward": 2.404787063598633,
+      "reward_std": 0.34057241678237915,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.22021275758743286,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 351
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 70.4,
+      "grad_norm": 0.5014784932136536,
+      "kl": 0.29856839776039124,
+      "learning_rate": 4.783863644106502e-06,
+      "loss": 0.0119,
+      "reward": 0.22824877500534058,
+      "reward_std": 1.6301518678665161,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8999999761581421,
+      "rewards/wrapped_driving_reward": -2.546751022338867,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 352
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 70.6,
+      "grad_norm": 0.4747620224952698,
+      "kl": 0.9486736059188843,
+      "learning_rate": 4.781639806422699e-06,
+      "loss": 0.0379,
+      "reward": 3.8079710006713867,
+      "reward_std": 0.04277324676513672,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9791666865348816,
+      "rewards/wrapped_driving_reward": 0.8288043737411499,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 353
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 70.8,
+      "grad_norm": 0.6246031522750854,
+      "kl": 0.6103500127792358,
+      "learning_rate": 4.779405108942722e-06,
+      "loss": 0.0244,
+      "reward": 3.2111012935638428,
+      "reward_std": 0.5528056621551514,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.46110111474990845,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 354
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 71.0,
+      "grad_norm": 0.7725012898445129,
+      "kl": 1.2677111625671387,
+      "learning_rate": 4.77715956230294e-06,
+      "loss": 0.0507,
+      "reward": -0.5685252547264099,
+      "reward_std": 1.8629494905471802,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -3.0685253143310547,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 355
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 71.2,
+      "grad_norm": 0.8144864439964294,
+      "kl": 0.9257476329803467,
+      "learning_rate": 4.774903177191358e-06,
+      "loss": 0.037,
+      "reward": 1.7669270038604736,
+      "reward_std": 3.5174825191497803,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.48307299613952637,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 356
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 71.4,
+      "grad_norm": 0.6923106908798218,
+      "kl": 0.52337646484375,
+      "learning_rate": 4.77263596434757e-06,
+      "loss": 0.0209,
+      "reward": -1.5257434844970703,
+      "reward_std": 3.4870338439941406,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.4583333134651184,
+      "rewards/wrapped_driving_reward": -2.859076738357544,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 357
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 71.6,
+      "grad_norm": 0.5352271199226379,
+      "kl": 0.8994762301445007,
+      "learning_rate": 4.770357934562704e-06,
+      "loss": 0.036,
+      "reward": 2.3911008834838867,
+      "reward_std": 0.5525059103965759,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.875,
+      "rewards/wrapped_driving_reward": -0.3588991165161133,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 358
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 71.8,
+      "grad_norm": 0.5663076043128967,
+      "kl": 0.7645798325538635,
+      "learning_rate": 4.7680690986793734e-06,
+      "loss": 0.0306,
+      "reward": 1.1860283613204956,
+      "reward_std": 3.1243858337402344,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.8139715790748596,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 359
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 72.0,
+      "grad_norm": 0.45141172409057617,
+      "kl": 0.5153231620788574,
+      "learning_rate": 4.765769467591626e-06,
+      "loss": 0.0206,
+      "reward": 2.5078225135803223,
+      "reward_std": 0.24923977255821228,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.3671773672103882,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 360
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 72.2,
+      "grad_norm": 0.4486269950866699,
+      "kl": 0.9089427590370178,
+      "learning_rate": 4.7634590522448886e-06,
+      "loss": 0.0364,
+      "reward": 2.8786072731018066,
+      "reward_std": 0.16827794909477234,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.121392622590065,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 361
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 72.4,
+      "grad_norm": 0.5474759340286255,
+      "kl": 0.8071030974388123,
+      "learning_rate": 4.761137863635921e-06,
+      "loss": 0.0323,
+      "reward": 2.189277172088623,
+      "reward_std": 0.3206322491168976,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.31072288751602173,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 362
+    },
+    {
+      "completion_length": 466.0,
+      "epoch": 72.6,
+      "grad_norm": 1.6158084869384766,
+      "kl": 0.6787058711051941,
+      "learning_rate": 4.758805912812755e-06,
+      "loss": 0.0271,
+      "reward": 3.3123486042022705,
+      "reward_std": 0.5808764100074768,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8123486042022705,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 363
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 72.8,
+      "grad_norm": 0.5949323177337646,
+      "kl": 0.9953030943870544,
+      "learning_rate": 4.7564632108746524e-06,
+      "loss": 0.0398,
+      "reward": 1.4721925258636475,
+      "reward_std": 2.548034906387329,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.699999988079071,
+      "rewards/wrapped_driving_reward": -0.7278074026107788,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 364
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 73.0,
+      "grad_norm": 0.5318465232849121,
+      "kl": 0.7464905977249146,
+      "learning_rate": 4.75410976897204e-06,
+      "loss": 0.0299,
+      "reward": 0.9842851758003235,
+      "reward_std": 2.294755697250366,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.8907147645950317,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 365
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 73.2,
+      "grad_norm": 0.6238420009613037,
+      "kl": 0.9425249099731445,
+      "learning_rate": 4.7517455983064694e-06,
+      "loss": 0.0377,
+      "reward": 3.152679204940796,
+      "reward_std": 0.3899010717868805,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.2776792049407959,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 366
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 73.4,
+      "grad_norm": 0.5412909984588623,
+      "kl": 0.31825780868530273,
+      "learning_rate": 4.7493707101305545e-06,
+      "loss": 0.0127,
+      "reward": -0.11978721618652344,
+      "reward_std": 1.6915035247802734,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -2.9947872161865234,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 367
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 73.6,
+      "grad_norm": 1.3431739807128906,
+      "kl": 0.5796495079994202,
+      "learning_rate": 4.746985115747918e-06,
+      "loss": 0.0232,
+      "reward": 2.067376136779785,
+      "reward_std": 2.382765769958496,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9861111044883728,
+      "rewards/wrapped_driving_reward": -0.6687348484992981,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 368
+    },
+    {
+      "completion_length": 467.0,
+      "epoch": 73.8,
+      "grad_norm": 0.5503766536712646,
+      "kl": 0.6919428706169128,
+      "learning_rate": 4.744588826513145e-06,
+      "loss": 0.0277,
+      "reward": 2.56295108795166,
+      "reward_std": 0.2011384665966034,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.918749988079071,
+      "rewards/wrapped_driving_reward": -0.35579875111579895,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 369
+    },
+    {
+      "completion_length": 407.0,
+      "epoch": 74.0,
+      "grad_norm": 0.5191554427146912,
+      "kl": 0.5616594552993774,
+      "learning_rate": 4.742181853831721e-06,
+      "loss": 0.0225,
+      "reward": 3.3623037338256836,
+      "reward_std": 0.5844976902008057,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.875,
+      "rewards/wrapped_driving_reward": 0.48730385303497314,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 370
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 74.2,
+      "grad_norm": 0.6369850635528564,
+      "kl": 0.351097047328949,
+      "learning_rate": 4.739764209159984e-06,
+      "loss": 0.014,
+      "reward": 0.09611350297927856,
+      "reward_std": 1.9310574531555176,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9147727489471436,
+      "rewards/wrapped_driving_reward": -2.3186590671539307,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 371
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 74.4,
+      "grad_norm": 0.592671275138855,
+      "kl": 0.7280439734458923,
+      "learning_rate": 4.737335904005063e-06,
+      "loss": 0.0291,
+      "reward": 2.9110307693481445,
+      "reward_std": 0.6375144124031067,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8863636255264282,
+      "rewards/wrapped_driving_reward": 0.3996671438217163,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 372
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 74.6,
+      "grad_norm": 0.5246638059616089,
+      "kl": 0.53719562292099,
+      "learning_rate": 4.734896949924831e-06,
+      "loss": 0.0215,
+      "reward": -0.7122367024421692,
+      "reward_std": 2.786146402359009,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -2.4622366428375244,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 373
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 74.8,
+      "grad_norm": 0.47917625308036804,
+      "kl": 0.8582153916358948,
+      "learning_rate": 4.732447358527843e-06,
+      "loss": 0.0343,
+      "reward": 0.6898465156555176,
+      "reward_std": 2.585700750350952,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9642857313156128,
+      "rewards/wrapped_driving_reward": -2.0244390964508057,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 374
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 75.0,
+      "grad_norm": 2.939560890197754,
+      "kl": 1.2595782279968262,
+      "learning_rate": 4.729987141473286e-06,
+      "loss": 0.0504,
+      "reward": 1.0538822412490845,
+      "reward_std": 3.369788885116577,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7083333134651184,
+      "rewards/wrapped_driving_reward": -1.1544511318206787,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 375
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 75.2,
+      "grad_norm": 0.5623316168785095,
+      "kl": 0.8516190648078918,
+      "learning_rate": 4.72751631047092e-06,
+      "loss": 0.0341,
+      "reward": 1.0435024499893188,
+      "reward_std": 1.7953479290008545,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8928571343421936,
+      "rewards/wrapped_driving_reward": -1.3493547439575195,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 376
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 75.4,
+      "grad_norm": 0.5572675466537476,
+      "kl": 0.5570351481437683,
+      "learning_rate": 4.725034877281025e-06,
+      "loss": 0.0223,
+      "reward": 2.0858306884765625,
+      "reward_std": 2.7353217601776123,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.539169430732727,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 377
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 75.6,
+      "grad_norm": 0.48211756348609924,
+      "kl": 0.8747799396514893,
+      "learning_rate": 4.7225428537143414e-06,
+      "loss": 0.035,
+      "reward": 2.519843578338623,
+      "reward_std": 0.06863429397344589,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.4801563024520874,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 378
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 75.8,
+      "grad_norm": 0.5445191860198975,
+      "kl": 0.5903018116950989,
+      "learning_rate": 4.720040251632019e-06,
+      "loss": 0.0236,
+      "reward": 2.2911336421966553,
+      "reward_std": 0.9245793223381042,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.33386632800102234,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 379
+    },
+    {
+      "completion_length": 436.0,
+      "epoch": 76.0,
+      "grad_norm": 0.494449645280838,
+      "kl": 1.1676191091537476,
+      "learning_rate": 4.717527082945555e-06,
+      "loss": 0.0467,
+      "reward": 3.4632315635681152,
+      "reward_std": 0.11494097858667374,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9583333134651184,
+      "rewards/wrapped_driving_reward": 0.5048981308937073,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 380
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 76.2,
+      "grad_norm": 0.5274999141693115,
+      "kl": 0.5592718124389648,
+      "learning_rate": 4.715003359616741e-06,
+      "loss": 0.0224,
+      "reward": 1.0327037572860718,
+      "reward_std": 2.441744804382324,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9642857313156128,
+      "rewards/wrapped_driving_reward": -1.931581974029541,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 381
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 76.4,
+      "grad_norm": 0.4961002469062805,
+      "kl": 0.4407399296760559,
+      "learning_rate": 4.712469093657605e-06,
+      "loss": 0.0176,
+      "reward": 0.06372499465942383,
+      "reward_std": 4.116382598876953,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.6862750053405762,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 382
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 76.6,
+      "grad_norm": 0.5338679552078247,
+      "kl": 0.6065806746482849,
+      "learning_rate": 4.709924297130354e-06,
+      "loss": 0.0243,
+      "reward": 0.48900270462036133,
+      "reward_std": 2.7124648094177246,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.6359972953796387,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 383
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 76.8,
+      "grad_norm": 0.5038735866546631,
+      "kl": 0.37462395429611206,
+      "learning_rate": 4.707368982147318e-06,
+      "loss": 0.015,
+      "reward": 1.0094149112701416,
+      "reward_std": 3.141808271408081,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.6428571343421936,
+      "rewards/wrapped_driving_reward": -0.758442223072052,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 384
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 77.0,
+      "grad_norm": 0.5533128976821899,
+      "kl": 0.5821331143379211,
+      "learning_rate": 4.704803160870888e-06,
+      "loss": 0.0233,
+      "reward": 3.293281078338623,
+      "reward_std": 0.5485936999320984,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.6682810187339783,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 385
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 77.2,
+      "grad_norm": 0.5457403659820557,
+      "kl": 0.8627141118049622,
+      "learning_rate": 4.702226845513465e-06,
+      "loss": 0.0345,
+      "reward": 2.2821011543273926,
+      "reward_std": 0.8117403984069824,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9090908765792847,
+      "rewards/wrapped_driving_reward": -0.12698988616466522,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 386
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 77.4,
+      "grad_norm": 0.4861312210559845,
+      "kl": 0.9649275541305542,
+      "learning_rate": 4.699640048337394e-06,
+      "loss": 0.0386,
+      "reward": 3.2577872276306152,
+      "reward_std": 0.1188662126660347,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9375,
+      "rewards/wrapped_driving_reward": 0.8202871084213257,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 387
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 77.6,
+      "grad_norm": 0.5439221858978271,
+      "kl": 1.217611312866211,
+      "learning_rate": 4.697042781654913e-06,
+      "loss": 0.0487,
+      "reward": 2.2471086978912354,
+      "reward_std": 2.214808225631714,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.6278913021087646,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 388
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 77.8,
+      "grad_norm": 0.6984148621559143,
+      "kl": 0.6679953932762146,
+      "learning_rate": 4.694435057828092e-06,
+      "loss": 0.0267,
+      "reward": 1.991325855255127,
+      "reward_std": 0.5920568704605103,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.7586740851402283,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 389
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 78.0,
+      "grad_norm": 0.5098196268081665,
+      "kl": 1.2996803522109985,
+      "learning_rate": 4.69181688926877e-06,
+      "loss": 0.052,
+      "reward": 3.0302999019622803,
+      "reward_std": 0.24492628872394562,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.28029996156692505,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 390
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 78.2,
+      "grad_norm": 0.7863979935646057,
+      "kl": 1.2847576141357422,
+      "learning_rate": 4.6891882884384994e-06,
+      "loss": 0.0514,
+      "reward": 1.7419676780700684,
+      "reward_std": 3.181126832962036,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7083333134651184,
+      "rewards/wrapped_driving_reward": -0.7163656949996948,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 391
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 78.4,
+      "grad_norm": 0.8814217448234558,
+      "kl": 1.0120067596435547,
+      "learning_rate": 4.68654926784849e-06,
+      "loss": 0.0405,
+      "reward": -1.4409170150756836,
+      "reward_std": 3.4238805770874023,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -2.9409170150756836,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 392
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 78.6,
+      "grad_norm": 1.0240262746810913,
+      "kl": 1.0795583724975586,
+      "learning_rate": 4.683899840059543e-06,
+      "loss": 0.0432,
+      "reward": 1.3624104261398315,
+      "reward_std": 2.9232280254364014,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.0125895738601685,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 393
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 78.8,
+      "grad_norm": 0.5109947919845581,
+      "kl": 0.2543693482875824,
+      "learning_rate": 4.681240017681994e-06,
+      "loss": 0.0102,
+      "reward": 1.1824758052825928,
+      "reward_std": 3.17807674407959,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9425241351127625,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 394
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 79.0,
+      "grad_norm": 0.7424824833869934,
+      "kl": 0.7182059288024902,
+      "learning_rate": 4.678569813375654e-06,
+      "loss": 0.0287,
+      "reward": 2.9500770568847656,
+      "reward_std": 0.5127381682395935,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9750000238418579,
+      "rewards/wrapped_driving_reward": 0.35007697343826294,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 395
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 79.2,
+      "grad_norm": 0.879162609577179,
+      "kl": 0.8275773525238037,
+      "learning_rate": 4.675889239849749e-06,
+      "loss": 0.0331,
+      "reward": 3.0128397941589355,
+      "reward_std": 0.4457358717918396,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.3878398835659027,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 396
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 79.4,
+      "grad_norm": 1.017330527305603,
+      "kl": 0.7725871801376343,
+      "learning_rate": 4.67319830986286e-06,
+      "loss": 0.0309,
+      "reward": 1.7406187057495117,
+      "reward_std": 3.501706838607788,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7083333134651184,
+      "rewards/wrapped_driving_reward": -0.46771466732025146,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 397
+    },
+    {
+      "completion_length": 474.0,
+      "epoch": 79.6,
+      "grad_norm": 0.6759405732154846,
+      "kl": 0.613670289516449,
+      "learning_rate": 4.670497036222856e-06,
+      "loss": 0.0245,
+      "reward": 3.172016143798828,
+      "reward_std": 0.8060486316680908,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9166666865348816,
+      "rewards/wrapped_driving_reward": 0.3803495764732361,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 398
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 79.8,
+      "grad_norm": 0.6927025318145752,
+      "kl": 0.3530423045158386,
+      "learning_rate": 4.667785431786843e-06,
+      "loss": 0.0141,
+      "reward": 2.0550060272216797,
+      "reward_std": 0.7213863134384155,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8531249761581421,
+      "rewards/wrapped_driving_reward": -0.29811891913414,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 399
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 80.0,
+      "grad_norm": 0.6842610836029053,
+      "kl": 0.3291209936141968,
+      "learning_rate": 4.665063509461098e-06,
+      "loss": 0.0132,
+      "reward": -0.5214939117431641,
+      "reward_std": 4.023183822631836,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -2.021493911743164,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 400
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 80.2,
+      "grad_norm": 1.280340552330017,
+      "kl": 0.9234899878501892,
+      "learning_rate": 4.662331282201002e-06,
+      "loss": 0.0369,
+      "reward": 1.608371615409851,
+      "reward_std": 3.448504686355591,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.6875,
+      "rewards/wrapped_driving_reward": -0.7041283845901489,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 401
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 80.4,
+      "grad_norm": 1.4907759428024292,
+      "kl": 0.787459135055542,
+      "learning_rate": 4.65958876301099e-06,
+      "loss": 0.0315,
+      "reward": 3.0962343215942383,
+      "reward_std": 0.35777872800827026,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.22123444080352783,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 402
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 80.6,
+      "grad_norm": 0.49698302149772644,
+      "kl": 0.24020951986312866,
+      "learning_rate": 4.65683596494448e-06,
+      "loss": 0.0096,
+      "reward": 2.8298206329345703,
+      "reward_std": 0.41156643629074097,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.925000011920929,
+      "rewards/wrapped_driving_reward": -0.09517934173345566,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 403
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 80.8,
+      "grad_norm": 1.796094298362732,
+      "kl": 0.980272650718689,
+      "learning_rate": 4.654072901103815e-06,
+      "loss": 0.0392,
+      "reward": 2.3932039737701416,
+      "reward_std": 0.9440638422966003,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.356795996427536,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 404
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 81.0,
+      "grad_norm": 1.1823846101760864,
+      "kl": 0.7713034749031067,
+      "learning_rate": 4.651299584640198e-06,
+      "loss": 0.0309,
+      "reward": 2.2802748680114746,
+      "reward_std": 0.5997734665870667,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.3447251319885254,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 405
+    },
+    {
+      "completion_length": 449.0,
+      "epoch": 81.2,
+      "grad_norm": 0.6266138553619385,
+      "kl": 0.7948797345161438,
+      "learning_rate": 4.648516028753632e-06,
+      "loss": 0.0318,
+      "reward": 2.6703319549560547,
+      "reward_std": 0.31966376304626465,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.20466792583465576,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 406
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 81.4,
+      "grad_norm": 0.7062759399414062,
+      "kl": 0.5212233066558838,
+      "learning_rate": 4.645722246692856e-06,
+      "loss": 0.0208,
+      "reward": 1.2558926343917847,
+      "reward_std": 3.199486017227173,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.71875,
+      "rewards/wrapped_driving_reward": -0.7128572463989258,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 407
+    },
+    {
+      "completion_length": 480.0,
+      "epoch": 81.6,
+      "grad_norm": 0.5641493201255798,
+      "kl": 0.8769745826721191,
+      "learning_rate": 4.642918251755281e-06,
+      "loss": 0.0351,
+      "reward": 2.6268346309661865,
+      "reward_std": 0.37705758213996887,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.0018346160650253296,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 408
+    },
+    {
+      "completion_length": 428.0,
+      "epoch": 81.8,
+      "grad_norm": 0.6789681315422058,
+      "kl": 0.580537736415863,
+      "learning_rate": 4.6401040572869295e-06,
+      "loss": 0.0232,
+      "reward": 3.1963601112365723,
+      "reward_std": 0.6234812140464783,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": 0.8213601112365723,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 409
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 82.0,
+      "grad_norm": 0.5842041373252869,
+      "kl": 0.9948064088821411,
+      "learning_rate": 4.637279676682367e-06,
+      "loss": 0.0398,
+      "reward": 2.9659104347229004,
+      "reward_std": 0.20523911714553833,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.21591025590896606,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 410
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 82.2,
+      "grad_norm": 0.7610025405883789,
+      "kl": 0.992956280708313,
+      "learning_rate": 4.634445123384644e-06,
+      "loss": 0.0397,
+      "reward": 1.2791435718536377,
+      "reward_std": 3.190803050994873,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9708565473556519,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 411
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 82.4,
+      "grad_norm": 0.5706319212913513,
+      "kl": 1.0067870616912842,
+      "learning_rate": 4.631600410885231e-06,
+      "loss": 0.0403,
+      "reward": 2.5278797149658203,
+      "reward_std": 0.3967418074607849,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9772727489471436,
+      "rewards/wrapped_driving_reward": -0.19939307868480682,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 412
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 82.6,
+      "grad_norm": 0.5617349147796631,
+      "kl": 0.8165132999420166,
+      "learning_rate": 4.6287455527239475e-06,
+      "loss": 0.0327,
+      "reward": 2.4629299640655518,
+      "reward_std": 0.6600992679595947,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9375,
+      "rewards/wrapped_driving_reward": 0.27542999386787415,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 413
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 82.8,
+      "grad_norm": 0.6648067831993103,
+      "kl": 0.639345109462738,
+      "learning_rate": 4.625880562488908e-06,
+      "loss": 0.0256,
+      "reward": 3.1534743309020996,
+      "reward_std": 0.5813019871711731,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.96875,
+      "rewards/wrapped_driving_reward": 0.5597245097160339,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 414
+    },
+    {
+      "completion_length": 482.0,
+      "epoch": 83.0,
+      "grad_norm": 0.5819172263145447,
+      "kl": 0.48156633973121643,
+      "learning_rate": 4.623005453816447e-06,
+      "loss": 0.0193,
+      "reward": 0.7172784805297852,
+      "reward_std": 2.3905746936798096,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.9077215194702148,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 415
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 83.2,
+      "grad_norm": 0.6890819668769836,
+      "kl": 0.970129132270813,
+      "learning_rate": 4.620120240391065e-06,
+      "loss": 0.0388,
+      "reward": 3.3380627632141113,
+      "reward_std": 0.025144066661596298,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.7130628824234009,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 416
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 83.4,
+      "grad_norm": 0.5055440068244934,
+      "kl": 1.0465319156646729,
+      "learning_rate": 4.617224935945354e-06,
+      "loss": 0.0419,
+      "reward": 1.7734256982803345,
+      "reward_std": 1.9043101072311401,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.1015743017196655,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 417
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 83.6,
+      "grad_norm": 0.7463017106056213,
+      "kl": 0.7646268606185913,
+      "learning_rate": 4.614319554259934e-06,
+      "loss": 0.0306,
+      "reward": 0.765255331993103,
+      "reward_std": 3.2255241870880127,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.6818181872367859,
+      "rewards/wrapped_driving_reward": -1.166562795639038,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 418
+    },
+    {
+      "completion_length": 494.0,
+      "epoch": 83.8,
+      "grad_norm": 0.613161027431488,
+      "kl": 1.2434450387954712,
+      "learning_rate": 4.611404109163392e-06,
+      "loss": 0.0497,
+      "reward": 3.17510986328125,
+      "reward_std": 0.35877394676208496,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9583333134651184,
+      "rewards/wrapped_driving_reward": 0.34177637100219727,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 419
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 84.0,
+      "grad_norm": 0.4756300151348114,
+      "kl": 0.8438820838928223,
+      "learning_rate": 4.608478614532215e-06,
+      "loss": 0.0338,
+      "reward": 2.6704771518707275,
+      "reward_std": 1.0551282167434692,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.20452289283275604,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 420
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 84.2,
+      "grad_norm": 0.6265870332717896,
+      "kl": 0.7220280170440674,
+      "learning_rate": 4.605543084290716e-06,
+      "loss": 0.0289,
+      "reward": 3.330387592315674,
+      "reward_std": 0.5945422053337097,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8303877115249634,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 421
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 84.4,
+      "grad_norm": 0.4972628653049469,
+      "kl": 0.4299587309360504,
+      "learning_rate": 4.602597532410982e-06,
+      "loss": 0.0172,
+      "reward": 1.6803350448608398,
+      "reward_std": 2.1513352394104004,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.9446649551391602,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 422
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 84.6,
+      "grad_norm": 1.2557274103164673,
+      "kl": 1.1829395294189453,
+      "learning_rate": 4.599641972912791e-06,
+      "loss": 0.0473,
+      "reward": 3.656310796737671,
+      "reward_std": 0.22195641696453094,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.7813107967376709,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 423
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 84.8,
+      "grad_norm": 0.5427276492118835,
+      "kl": 0.6175304055213928,
+      "learning_rate": 4.596676419863561e-06,
+      "loss": 0.0247,
+      "reward": 2.078035593032837,
+      "reward_std": 1.5830367803573608,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.7969645261764526,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 424
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 85.0,
+      "grad_norm": 0.6221134066581726,
+      "kl": 0.36543771624565125,
+      "learning_rate": 4.59370088737827e-06,
+      "loss": 0.0146,
+      "reward": 3.0459718704223633,
+      "reward_std": 0.3525036573410034,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9750000238418579,
+      "rewards/wrapped_driving_reward": 0.19597166776657104,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 425
+    },
+    {
+      "completion_length": 471.0,
+      "epoch": 85.2,
+      "grad_norm": 1.1358462572097778,
+      "kl": 1.153946042060852,
+      "learning_rate": 4.590715389619399e-06,
+      "loss": 0.0462,
+      "reward": 2.3951056003570557,
+      "reward_std": 0.31768667697906494,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.22989457845687866,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 426
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 85.4,
+      "grad_norm": 0.6305968761444092,
+      "kl": 0.5702832341194153,
+      "learning_rate": 4.587719940796858e-06,
+      "loss": 0.0228,
+      "reward": 1.2231757640838623,
+      "reward_std": 1.9306583404541016,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.5268242359161377,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 427
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 85.6,
+      "grad_norm": 0.8879334330558777,
+      "kl": 0.9042296409606934,
+      "learning_rate": 4.584714555167921e-06,
+      "loss": 0.0362,
+      "reward": 3.215460777282715,
+      "reward_std": 0.508735716342926,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9444444179534912,
+      "rewards/wrapped_driving_reward": 0.39601635932922363,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 428
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 85.8,
+      "grad_norm": 0.4813222289085388,
+      "kl": 0.408966988325119,
+      "learning_rate": 4.581699247037157e-06,
+      "loss": 0.0164,
+      "reward": 2.7956860065460205,
+      "reward_std": 0.7762725353240967,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9285714626312256,
+      "rewards/wrapped_driving_reward": 0.11711461842060089,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 429
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 86.0,
+      "grad_norm": 0.5622531771659851,
+      "kl": 0.9757765531539917,
+      "learning_rate": 4.578674030756364e-06,
+      "loss": 0.039,
+      "reward": 3.035037040710449,
+      "reward_std": 0.4434798061847687,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.949999988079071,
+      "rewards/wrapped_driving_reward": 0.21003687381744385,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 430
+    },
+    {
+      "completion_length": 488.0,
+      "epoch": 86.2,
+      "grad_norm": 0.499776154756546,
+      "kl": 1.332625389099121,
+      "learning_rate": 4.5756389207244965e-06,
+      "loss": 0.0533,
+      "reward": 3.2134695053100586,
+      "reward_std": 0.827461302280426,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9375,
+      "rewards/wrapped_driving_reward": 0.5259695053100586,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 431
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 86.4,
+      "grad_norm": 0.4789699912071228,
+      "kl": 0.47637441754341125,
+      "learning_rate": 4.572593931387604e-06,
+      "loss": 0.0191,
+      "reward": 2.457927703857422,
+      "reward_std": 0.9853748083114624,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9375,
+      "rewards/wrapped_driving_reward": -0.3545722961425781,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 432
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 86.6,
+      "grad_norm": 0.4391123354434967,
+      "kl": 1.1674070358276367,
+      "learning_rate": 4.569539077238756e-06,
+      "loss": 0.0467,
+      "reward": 2.6357340812683105,
+      "reward_std": 0.510990560054779,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.13573402166366577,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 433
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 86.8,
+      "grad_norm": 0.5584562420845032,
+      "kl": 0.7644482851028442,
+      "learning_rate": 4.566474372817971e-06,
+      "loss": 0.0306,
+      "reward": 2.366978645324707,
+      "reward_std": 0.4779915511608124,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.13302119076251984,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 434
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 87.0,
+      "grad_norm": 0.5053340792655945,
+      "kl": 0.5127562284469604,
+      "learning_rate": 4.5633998327121595e-06,
+      "loss": 0.0205,
+      "reward": 2.978019952774048,
+      "reward_std": 0.3388763666152954,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.02198006771504879,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 435
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 87.2,
+      "grad_norm": 0.5881168246269226,
+      "kl": 0.8654472231864929,
+      "learning_rate": 4.560315471555039e-06,
+      "loss": 0.0346,
+      "reward": 2.695511817932129,
+      "reward_std": 0.2650168240070343,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.17948828637599945,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 436
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 87.4,
+      "grad_norm": 0.5051465630531311,
+      "kl": 0.7103445529937744,
+      "learning_rate": 4.557221304027077e-06,
+      "loss": 0.0284,
+      "reward": 0.450472354888916,
+      "reward_std": 2.2633330821990967,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.924527645111084,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 437
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 87.6,
+      "grad_norm": 0.8089684247970581,
+      "kl": 0.7311402559280396,
+      "learning_rate": 4.55411734485541e-06,
+      "loss": 0.0292,
+      "reward": 3.2812561988830566,
+      "reward_std": 0.32883548736572266,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.4062563478946686,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 438
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 87.8,
+      "grad_norm": 0.5592673420906067,
+      "kl": 0.6468991041183472,
+      "learning_rate": 4.551003608813784e-06,
+      "loss": 0.0259,
+      "reward": 1.8620704412460327,
+      "reward_std": 0.6889193058013916,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9750000238418579,
+      "rewards/wrapped_driving_reward": -0.8629295229911804,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 439
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 88.0,
+      "grad_norm": 0.5054028630256653,
+      "kl": 0.36086270213127136,
+      "learning_rate": 4.54788011072248e-06,
+      "loss": 0.0144,
+      "reward": 3.3388941287994385,
+      "reward_std": 0.5939581394195557,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8388940691947937,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 440
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 88.2,
+      "grad_norm": 9.32406997680664,
+      "kl": 2.2713303565979004,
+      "learning_rate": 4.544746865448239e-06,
+      "loss": 0.0909,
+      "reward": 1.8941614627838135,
+      "reward_std": 2.3242433071136475,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.7308385968208313,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 441
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 88.4,
+      "grad_norm": 0.7286486625671387,
+      "kl": 0.23764638602733612,
+      "learning_rate": 4.541603887904198e-06,
+      "loss": 0.0095,
+      "reward": 1.595595121383667,
+      "reward_std": 3.398191213607788,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.65625,
+      "rewards/wrapped_driving_reward": -0.43565481901168823,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 442
+    },
+    {
+      "completion_length": 472.0,
+      "epoch": 88.6,
+      "grad_norm": 0.4791865944862366,
+      "kl": 0.853591799736023,
+      "learning_rate": 4.538451193049814e-06,
+      "loss": 0.0341,
+      "reward": 2.5504026412963867,
+      "reward_std": 0.576043963432312,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.4495972692966461,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 443
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 88.8,
+      "grad_norm": 0.5535397529602051,
+      "kl": 0.1802796721458435,
+      "learning_rate": 4.535288795890799e-06,
+      "loss": 0.0072,
+      "reward": -0.1438617706298828,
+      "reward_std": 3.8885178565979004,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.44999998807907104,
+      "rewards/wrapped_driving_reward": -1.593861699104309,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 444
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 89.0,
+      "grad_norm": 0.5480430126190186,
+      "kl": 0.5194934010505676,
+      "learning_rate": 4.532116711479039e-06,
+      "loss": 0.0208,
+      "reward": 2.7844161987304688,
+      "reward_std": 0.34969019889831543,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.0905836746096611,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 445
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 89.2,
+      "grad_norm": 7.007938861846924,
+      "kl": 1.402961015701294,
+      "learning_rate": 4.528934954912531e-06,
+      "loss": 0.0561,
+      "reward": -0.7073632478713989,
+      "reward_std": 2.505068778991699,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7083333134651184,
+      "rewards/wrapped_driving_reward": -3.040696620941162,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 446
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 89.4,
+      "grad_norm": 0.558776319026947,
+      "kl": 1.3707541227340698,
+      "learning_rate": 4.525743541335309e-06,
+      "loss": 0.0548,
+      "reward": 2.7044730186462402,
+      "reward_std": 0.5199458003044128,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.2955269515514374,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 447
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 89.6,
+      "grad_norm": 0.46237713098526,
+      "kl": 0.986226499080658,
+      "learning_rate": 4.522542485937369e-06,
+      "loss": 0.0394,
+      "reward": 3.5677292346954346,
+      "reward_std": 0.30159273743629456,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.817729115486145,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 448
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 89.8,
+      "grad_norm": 0.6039856672286987,
+      "kl": 0.5541295409202576,
+      "learning_rate": 4.519331803954599e-06,
+      "loss": 0.0222,
+      "reward": 2.4867310523986816,
+      "reward_std": 0.18556839227676392,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9147727489471436,
+      "rewards/wrapped_driving_reward": -0.17804156243801117,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 449
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 90.0,
+      "grad_norm": 0.6416642665863037,
+      "kl": 0.5888490676879883,
+      "learning_rate": 4.516111510668707e-06,
+      "loss": 0.0236,
+      "reward": 3.0717105865478516,
+      "reward_std": 0.9419719576835632,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.5717105865478516,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 450
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 90.2,
+      "grad_norm": 0.8580331802368164,
+      "kl": 1.3547351360321045,
+      "learning_rate": 4.512881621407146e-06,
+      "loss": 0.0542,
+      "reward": 1.4438624382019043,
+      "reward_std": 3.3256571292877197,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9311375021934509,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 451
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 90.4,
+      "grad_norm": 0.6190384030342102,
+      "kl": 0.4166165888309479,
+      "learning_rate": 4.509642151543043e-06,
+      "loss": 0.0167,
+      "reward": 1.4482831954956055,
+      "reward_std": 3.3409366607666016,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9267167448997498,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 452
+    },
+    {
+      "completion_length": 470.0,
+      "epoch": 90.6,
+      "grad_norm": 0.9551142454147339,
+      "kl": 0.9795337915420532,
+      "learning_rate": 4.506393116495128e-06,
+      "loss": 0.0392,
+      "reward": 3.8107566833496094,
+      "reward_std": 0.042369965463876724,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9750000238418579,
+      "rewards/wrapped_driving_reward": 0.835756778717041,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 453
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 90.8,
+      "grad_norm": 0.532974362373352,
+      "kl": 0.43851980566978455,
+      "learning_rate": 4.503134531727652e-06,
+      "loss": 0.0175,
+      "reward": 2.5775976181030273,
+      "reward_std": 0.30674323439598083,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.17240235209465027,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 454
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 91.0,
+      "grad_norm": 0.5811918377876282,
+      "kl": 0.46373996138572693,
+      "learning_rate": 4.499866412750324e-06,
+      "loss": 0.0185,
+      "reward": 3.5562117099761963,
+      "reward_std": 0.12775082886219025,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.5562118291854858,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 455
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 91.2,
+      "grad_norm": 0.53252774477005,
+      "kl": 0.653654158115387,
+      "learning_rate": 4.496588775118232e-06,
+      "loss": 0.0261,
+      "reward": 1.345137119293213,
+      "reward_std": 1.6006327867507935,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9750000238418579,
+      "rewards/wrapped_driving_reward": -1.2548627853393555,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 456
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 91.4,
+      "grad_norm": 0.6192057728767395,
+      "kl": 0.8161628842353821,
+      "learning_rate": 4.493301634431768e-06,
+      "loss": 0.0326,
+      "reward": 3.238480567932129,
+      "reward_std": 0.6025396585464478,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9750000238418579,
+      "rewards/wrapped_driving_reward": 0.513480544090271,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 457
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 91.6,
+      "grad_norm": 0.5338960289955139,
+      "kl": 0.4019594192504883,
+      "learning_rate": 4.490005006336555e-06,
+      "loss": 0.0161,
+      "reward": 1.511120080947876,
+      "reward_std": 3.7101433277130127,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.6138800382614136,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 458
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 91.8,
+      "grad_norm": 0.6177345514297485,
+      "kl": 0.7023136615753174,
+      "learning_rate": 4.486698906523375e-06,
+      "loss": 0.0281,
+      "reward": 0.6668235063552856,
+      "reward_std": 2.7802770137786865,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7022727131843567,
+      "rewards/wrapped_driving_reward": -1.1604492664337158,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 459
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 92.0,
+      "grad_norm": 0.498141884803772,
+      "kl": 0.7440797686576843,
+      "learning_rate": 4.4833833507280884e-06,
+      "loss": 0.0298,
+      "reward": 1.770745038986206,
+      "reward_std": 2.273341178894043,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.854254961013794,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 460
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 92.2,
+      "grad_norm": 0.5320531725883484,
+      "kl": 0.5814719796180725,
+      "learning_rate": 4.4800583547315654e-06,
+      "loss": 0.0233,
+      "reward": -0.1779249906539917,
+      "reward_std": 1.991480827331543,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -3.0529251098632812,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 461
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 92.4,
+      "grad_norm": 0.6644296646118164,
+      "kl": 1.252918004989624,
+      "learning_rate": 4.476723934359609e-06,
+      "loss": 0.0501,
+      "reward": 1.581032395362854,
+      "reward_std": 3.7278060913085938,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.6689676642417908,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 462
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 92.6,
+      "grad_norm": 0.6294512748718262,
+      "kl": 0.454416960477829,
+      "learning_rate": 4.473380105482875e-06,
+      "loss": 0.0182,
+      "reward": 0.13677752017974854,
+      "reward_std": 2.944185733795166,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -2.113222360610962,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 463
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 92.8,
+      "grad_norm": 0.5402928590774536,
+      "kl": 0.8170625567436218,
+      "learning_rate": 4.470026884016805e-06,
+      "loss": 0.0327,
+      "reward": 3.565312385559082,
+      "reward_std": 0.3042621612548828,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8153125047683716,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 464
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 93.0,
+      "grad_norm": 0.533676266670227,
+      "kl": 0.7118498682975769,
+      "learning_rate": 4.466664285921543e-06,
+      "loss": 0.0285,
+      "reward": 2.770576000213623,
+      "reward_std": 0.16069670021533966,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9545454978942871,
+      "rewards/wrapped_driving_reward": -0.05896963179111481,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 465
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 93.2,
+      "grad_norm": 0.5372627377510071,
+      "kl": 1.061421275138855,
+      "learning_rate": 4.463292327201862e-06,
+      "loss": 0.0425,
+      "reward": 0.8557369112968445,
+      "reward_std": 2.914240598678589,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.3942632675170898,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 466
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 93.4,
+      "grad_norm": 0.6193510293960571,
+      "kl": 1.1515281200408936,
+      "learning_rate": 4.459911023907092e-06,
+      "loss": 0.0461,
+      "reward": 3.183587074279785,
+      "reward_std": 0.39359328150749207,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9472222328186035,
+      "rewards/wrapped_driving_reward": 0.4863646626472473,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 467
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 93.6,
+      "grad_norm": 0.5153403878211975,
+      "kl": 0.6276007890701294,
+      "learning_rate": 4.456520392131035e-06,
+      "loss": 0.0251,
+      "reward": 1.2580312490463257,
+      "reward_std": 2.2587177753448486,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.6169687509536743,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 468
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 93.8,
+      "grad_norm": 0.8933354616165161,
+      "kl": 0.5966981053352356,
+      "learning_rate": 4.453120448011897e-06,
+      "loss": 0.0239,
+      "reward": 3.828160285949707,
+      "reward_std": 0.019492290914058685,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.828160285949707,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 469
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 94.0,
+      "grad_norm": 0.5882790684700012,
+      "kl": 0.6073835492134094,
+      "learning_rate": 4.4497112077322045e-06,
+      "loss": 0.0243,
+      "reward": 2.9608585834503174,
+      "reward_std": 0.5855686664581299,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9791666865348816,
+      "rewards/wrapped_driving_reward": 0.2316918969154358,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 470
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 94.2,
+      "grad_norm": 0.5647352337837219,
+      "kl": 0.695755660533905,
+      "learning_rate": 4.446292687518734e-06,
+      "loss": 0.0278,
+      "reward": 1.5310817956924438,
+      "reward_std": 3.7384822368621826,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.6785714626312256,
+      "rewards/wrapped_driving_reward": -0.39748966693878174,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 471
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 94.4,
+      "grad_norm": 0.5346417427062988,
+      "kl": 0.8996866345405579,
+      "learning_rate": 4.442864903642428e-06,
+      "loss": 0.036,
+      "reward": 1.0604596138000488,
+      "reward_std": 3.060889959335327,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.625,
+      "rewards/wrapped_driving_reward": -1.0645403861999512,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 472
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 94.6,
+      "grad_norm": 0.5798156261444092,
+      "kl": 0.6437966227531433,
+      "learning_rate": 4.439427872418321e-06,
+      "loss": 0.0258,
+      "reward": 3.384183168411255,
+      "reward_std": 0.42769670486450195,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.5091832280158997,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 473
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 94.8,
+      "grad_norm": 0.49952396750450134,
+      "kl": 0.5218140482902527,
+      "learning_rate": 4.435981610205464e-06,
+      "loss": 0.0209,
+      "reward": -0.285653293132782,
+      "reward_std": 2.212059497833252,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9583333134651184,
+      "rewards/wrapped_driving_reward": -2.9939866065979004,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 474
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 95.0,
+      "grad_norm": 1.0312459468841553,
+      "kl": 0.6847164630889893,
+      "learning_rate": 4.432526133406843e-06,
+      "loss": 0.0274,
+      "reward": 2.879366636276245,
+      "reward_std": 0.39730778336524963,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.2543666362762451,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 475
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 95.2,
+      "grad_norm": 0.582595705986023,
+      "kl": 0.599985659122467,
+      "learning_rate": 4.4290614584693005e-06,
+      "loss": 0.024,
+      "reward": 0.7622314691543579,
+      "reward_std": 2.525625228881836,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.237768530845642,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 476
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 95.4,
+      "grad_norm": 0.5313723683357239,
+      "kl": 0.868817925453186,
+      "learning_rate": 4.425587601883461e-06,
+      "loss": 0.0348,
+      "reward": 2.211709976196289,
+      "reward_std": 0.4423002600669861,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.96875,
+      "rewards/wrapped_driving_reward": -0.3820401132106781,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 477
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 95.6,
+      "grad_norm": 0.5549596548080444,
+      "kl": 1.2767629623413086,
+      "learning_rate": 4.422104580183649e-06,
+      "loss": 0.0511,
+      "reward": 3.195434331893921,
+      "reward_std": 0.395979642868042,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.875,
+      "rewards/wrapped_driving_reward": 0.4454343914985657,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 478
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 95.8,
+      "grad_norm": 0.4778159260749817,
+      "kl": 0.23531889915466309,
+      "learning_rate": 4.418612409947814e-06,
+      "loss": 0.0094,
+      "reward": 2.747316360473633,
+      "reward_std": 0.6012102961540222,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.122316375374794,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 479
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 96.0,
+      "grad_norm": 0.6010138392448425,
+      "kl": 0.6477577090263367,
+      "learning_rate": 4.415111107797445e-06,
+      "loss": 0.0259,
+      "reward": 1.7465873956680298,
+      "reward_std": 3.5261001586914062,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.3784126043319702,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 480
+    },
+    {
+      "completion_length": 476.0,
+      "epoch": 96.2,
+      "grad_norm": 0.4925939440727234,
+      "kl": 1.1028554439544678,
+      "learning_rate": 4.4116006903975015e-06,
+      "loss": 0.0441,
+      "reward": 2.4060165882110596,
+      "reward_std": 0.4904329478740692,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.984375,
+      "rewards/wrapped_driving_reward": -0.20335841178894043,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 481
+    },
+    {
+      "completion_length": 467.0,
+      "epoch": 96.4,
+      "grad_norm": 0.5640705823898315,
+      "kl": 1.066386103630066,
+      "learning_rate": 4.408081174456322e-06,
+      "loss": 0.0427,
+      "reward": 3.762505292892456,
+      "reward_std": 0.12346359342336655,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9375,
+      "rewards/wrapped_driving_reward": 0.825005292892456,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 482
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 96.6,
+      "grad_norm": 0.5508888959884644,
+      "kl": 0.3224889039993286,
+      "learning_rate": 4.404552576725557e-06,
+      "loss": 0.0129,
+      "reward": 1.1607617139816284,
+      "reward_std": 3.1784019470214844,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.7142382860183716,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 483
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 96.8,
+      "grad_norm": 0.5421119332313538,
+      "kl": 0.2217361032962799,
+      "learning_rate": 4.401014914000078e-06,
+      "loss": 0.0089,
+      "reward": 1.3363996744155884,
+      "reward_std": 3.2391209602355957,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.7886003255844116,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 484
+    },
+    {
+      "completion_length": 403.0,
+      "epoch": 97.0,
+      "grad_norm": 0.5398365259170532,
+      "kl": 1.3086752891540527,
+      "learning_rate": 4.397468203117905e-06,
+      "loss": 0.0523,
+      "reward": 2.178347110748291,
+      "reward_std": 2.1330792903900146,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.571652889251709,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 485
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 97.2,
+      "grad_norm": 0.5164806246757507,
+      "kl": 1.1992006301879883,
+      "learning_rate": 4.393912460960125e-06,
+      "loss": 0.048,
+      "reward": -0.017039000988006592,
+      "reward_std": 1.9406579732894897,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.869949460029602,
+      "rewards/wrapped_driving_reward": -2.386988401412964,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 486
+    },
+    {
+      "completion_length": 447.0,
+      "epoch": 97.4,
+      "grad_norm": 0.5225284695625305,
+      "kl": 0.9045883417129517,
+      "learning_rate": 4.3903477044508066e-06,
+      "loss": 0.0362,
+      "reward": 3.6854019165039062,
+      "reward_std": 0.25948864221572876,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8104017972946167,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 487
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 97.6,
+      "grad_norm": 0.5833526253700256,
+      "kl": 0.5774132013320923,
+      "learning_rate": 4.386773950556931e-06,
+      "loss": 0.0231,
+      "reward": 3.013421058654785,
+      "reward_std": 0.4619700014591217,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9583333134651184,
+      "rewards/wrapped_driving_reward": 0.18008768558502197,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 488
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 97.8,
+      "grad_norm": 0.4881477653980255,
+      "kl": 0.38699668645858765,
+      "learning_rate": 4.3831912162882946e-06,
+      "loss": 0.0155,
+      "reward": 0.5115635395050049,
+      "reward_std": 1.8448858261108398,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8522727489471436,
+      "rewards/wrapped_driving_reward": -1.9657092094421387,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 489
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 98.0,
+      "grad_norm": 0.5169448852539062,
+      "kl": 1.3162471055984497,
+      "learning_rate": 4.379599518697444e-06,
+      "loss": 0.0526,
+      "reward": 3.255016803741455,
+      "reward_std": 0.4777297377586365,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.875,
+      "rewards/wrapped_driving_reward": 0.5050168633460999,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 490
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 98.2,
+      "grad_norm": 0.553719699382782,
+      "kl": 0.4232349097728729,
+      "learning_rate": 4.375998874879585e-06,
+      "loss": 0.0169,
+      "reward": -2.375,
+      "reward_std": 1.6007810831069946,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 491
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 98.4,
+      "grad_norm": 0.549466073513031,
+      "kl": 0.9794742465019226,
+      "learning_rate": 4.372389301972506e-06,
+      "loss": 0.0392,
+      "reward": 2.8539419174194336,
+      "reward_std": 0.11372269690036774,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.14605820178985596,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 492
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 98.6,
+      "grad_norm": 0.4947136342525482,
+      "kl": 0.7255305051803589,
+      "learning_rate": 4.368770817156493e-06,
+      "loss": 0.029,
+      "reward": 1.4876158237457275,
+      "reward_std": 3.332230567932129,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.3873841464519501,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 493
+    },
+    {
+      "completion_length": 480.0,
+      "epoch": 98.8,
+      "grad_norm": 0.60732501745224,
+      "kl": 0.8074931502342224,
+      "learning_rate": 4.365143437654249e-06,
+      "loss": 0.0323,
+      "reward": 3.7022383213043213,
+      "reward_std": 0.10065864771604538,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.96875,
+      "rewards/wrapped_driving_reward": 0.7334883213043213,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 494
+    },
+    {
+      "completion_length": 471.0,
+      "epoch": 99.0,
+      "grad_norm": 0.5607241988182068,
+      "kl": 0.9837010502815247,
+      "learning_rate": 4.3615071807308165e-06,
+      "loss": 0.0393,
+      "reward": 2.795694589614868,
+      "reward_std": 0.325857937335968,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.04569460451602936,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 495
+    },
+    {
+      "completion_length": 349.0,
+      "epoch": 99.2,
+      "grad_norm": 0.5761314034461975,
+      "kl": 0.4112725853919983,
+      "learning_rate": 4.357862063693486e-06,
+      "loss": 0.0165,
+      "reward": 3.6457560062408447,
+      "reward_std": 0.12520428001880646,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.984375,
+      "rewards/wrapped_driving_reward": 0.6613809466362,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 496
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 99.4,
+      "grad_norm": 0.4828655421733856,
+      "kl": 0.8910616636276245,
+      "learning_rate": 4.354208103891723e-06,
+      "loss": 0.0356,
+      "reward": 3.4025442600250244,
+      "reward_std": 0.550991415977478,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.96875,
+      "rewards/wrapped_driving_reward": 0.6837941408157349,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 497
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 99.6,
+      "grad_norm": 3.4703691005706787,
+      "kl": 0.9643664360046387,
+      "learning_rate": 4.350545318717081e-06,
+      "loss": 0.0386,
+      "reward": 2.7853288650512695,
+      "reward_std": 0.7604644894599915,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.03532897308468819,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 498
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 99.8,
+      "grad_norm": 0.5264486074447632,
+      "kl": 0.6139304041862488,
+      "learning_rate": 4.3468737256031155e-06,
+      "loss": 0.0246,
+      "reward": 1.6845433712005615,
+      "reward_std": 0.8301387429237366,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.1904566287994385,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 499
+    },
+    {
+      "completion_length": 419.0,
+      "epoch": 100.0,
+      "grad_norm": 0.5735057592391968,
+      "kl": 0.41647979617118835,
+      "learning_rate": 4.34319334202531e-06,
+      "loss": 0.0167,
+      "reward": 2.6967225074768066,
+      "reward_std": 0.0777384340763092,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9437500238418579,
+      "rewards/wrapped_driving_reward": -0.24702748656272888,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 500
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 100.2,
+      "grad_norm": 0.7586433291435242,
+      "kl": 0.7535019516944885,
+      "learning_rate": 4.339504185500984e-06,
+      "loss": 0.0301,
+      "reward": 3.8102352619171143,
+      "reward_std": 0.015372190624475479,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.810235321521759,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 501
+    },
+    {
+      "completion_length": 466.0,
+      "epoch": 100.4,
+      "grad_norm": 0.6329609751701355,
+      "kl": 1.2336125373840332,
+      "learning_rate": 4.335806273589214e-06,
+      "loss": 0.0493,
+      "reward": 1.1259068250656128,
+      "reward_std": 3.417491912841797,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.1240931749343872,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 502
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 100.6,
+      "grad_norm": 0.5284588932991028,
+      "kl": 1.2016366720199585,
+      "learning_rate": 4.332099623890749e-06,
+      "loss": 0.0481,
+      "reward": 2.9199860095977783,
+      "reward_std": 0.41995736956596375,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.04498597979545593,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 503
+    },
+    {
+      "completion_length": 371.0,
+      "epoch": 100.8,
+      "grad_norm": 0.6299885511398315,
+      "kl": 0.4117721617221832,
+      "learning_rate": 4.328384254047927e-06,
+      "loss": 0.0165,
+      "reward": 3.296032428741455,
+      "reward_std": 0.5489058494567871,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.5460324287414551,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 504
+    },
+    {
+      "completion_length": 487.0,
+      "epoch": 101.0,
+      "grad_norm": 0.5148439407348633,
+      "kl": 0.7591438889503479,
+      "learning_rate": 4.324660181744589e-06,
+      "loss": 0.0304,
+      "reward": 2.263432502746582,
+      "reward_std": 1.198183536529541,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.3615674376487732,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 505
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 101.2,
+      "grad_norm": 0.5383084416389465,
+      "kl": 0.8281757235527039,
+      "learning_rate": 4.320927424706001e-06,
+      "loss": 0.0331,
+      "reward": 2.926753520965576,
+      "reward_std": 0.5967720746994019,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.17675352096557617,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 506
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 101.4,
+      "grad_norm": 0.5339255332946777,
+      "kl": 0.8783241510391235,
+      "learning_rate": 4.317186000698761e-06,
+      "loss": 0.0351,
+      "reward": 2.7582178115844727,
+      "reward_std": 0.5562581419944763,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.2582179605960846,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 507
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 101.6,
+      "grad_norm": 0.5230175256729126,
+      "kl": 0.7820099592208862,
+      "learning_rate": 4.313435927530719e-06,
+      "loss": 0.0313,
+      "reward": 1.851464867591858,
+      "reward_std": 0.8467382788658142,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.6485350728034973,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 508
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 101.8,
+      "grad_norm": 0.4553928077220917,
+      "kl": 0.7339428067207336,
+      "learning_rate": 4.309677223050895e-06,
+      "loss": 0.0294,
+      "reward": 1.0629948377609253,
+      "reward_std": 3.375459671020508,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.1870051622390747,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 509
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 102.0,
+      "grad_norm": 6.9111127853393555,
+      "kl": 2.688588857650757,
+      "learning_rate": 4.305909905149389e-06,
+      "loss": 0.1075,
+      "reward": 1.5133618116378784,
+      "reward_std": 3.677501916885376,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.7366381287574768,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 510
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 102.2,
+      "grad_norm": 0.6580697298049927,
+      "kl": 1.4310871362686157,
+      "learning_rate": 4.3021339917572975e-06,
+      "loss": 0.0572,
+      "reward": 0.9614809155464172,
+      "reward_std": 3.113192558288574,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.1635191440582275,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 511
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 102.4,
+      "grad_norm": 0.5757476091384888,
+      "kl": 1.0578932762145996,
+      "learning_rate": 4.2983495008466285e-06,
+      "loss": 0.0423,
+      "reward": 1.5223838090896606,
+      "reward_std": 3.3499815464019775,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.8526161909103394,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 512
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 102.6,
+      "grad_norm": 0.5535104870796204,
+      "kl": 1.4569833278656006,
+      "learning_rate": 4.294556450430216e-06,
+      "loss": 0.0583,
+      "reward": 2.8807883262634277,
+      "reward_std": 0.7175414562225342,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.25578856468200684,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 513
+    },
+    {
+      "completion_length": 386.0,
+      "epoch": 102.8,
+      "grad_norm": 0.538215696811676,
+      "kl": 0.8643997311592102,
+      "learning_rate": 4.290754858561636e-06,
+      "loss": 0.0346,
+      "reward": 3.3438968658447266,
+      "reward_std": 0.5325138568878174,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9272727370262146,
+      "rewards/wrapped_driving_reward": 0.4166242182254791,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 514
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 103.0,
+      "grad_norm": 0.5776817798614502,
+      "kl": 1.1718462705612183,
+      "learning_rate": 4.2869447433351165e-06,
+      "loss": 0.0469,
+      "reward": 1.0269173383712769,
+      "reward_std": 3.018467426300049,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7272727489471436,
+      "rewards/wrapped_driving_reward": -1.3253554105758667,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 515
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 103.2,
+      "grad_norm": 0.5751096606254578,
+      "kl": 1.1135950088500977,
+      "learning_rate": 4.283126122885455e-06,
+      "loss": 0.0445,
+      "reward": 2.5814623832702637,
+      "reward_std": 0.2912139296531677,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.16853773593902588,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 516
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 103.4,
+      "grad_norm": 0.6097070574760437,
+      "kl": 1.033371090888977,
+      "learning_rate": 4.2792990153879286e-06,
+      "loss": 0.0413,
+      "reward": 3.570962905883789,
+      "reward_std": 0.47861868143081665,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8209629058837891,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 517
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 103.6,
+      "grad_norm": 0.5641130805015564,
+      "kl": 0.7364106774330139,
+      "learning_rate": 4.275463439058214e-06,
+      "loss": 0.0295,
+      "reward": 2.555635929107666,
+      "reward_std": 1.0865808725357056,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.06936385482549667,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 518
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 103.8,
+      "grad_norm": 0.8291902542114258,
+      "kl": 1.3542972803115845,
+      "learning_rate": 4.271619412152293e-06,
+      "loss": 0.0542,
+      "reward": 0.07205808162689209,
+      "reward_std": 3.21856689453125,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -2.0529417991638184,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 519
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 104.0,
+      "grad_norm": 0.497583270072937,
+      "kl": 1.1503536701202393,
+      "learning_rate": 4.267766952966369e-06,
+      "loss": 0.046,
+      "reward": 3.3464627265930176,
+      "reward_std": 0.23250795900821686,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.4714627265930176,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 520
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 104.2,
+      "grad_norm": 0.510185182094574,
+      "kl": 0.9102869033813477,
+      "learning_rate": 4.2639060798367835e-06,
+      "loss": 0.0364,
+      "reward": 3.5760674476623535,
+      "reward_std": 0.17436249554157257,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.96875,
+      "rewards/wrapped_driving_reward": 0.7323175668716431,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 521
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 104.4,
+      "grad_norm": 0.6613531708717346,
+      "kl": 0.6623646020889282,
+      "learning_rate": 4.260036811139922e-06,
+      "loss": 0.0265,
+      "reward": 0.3890085220336914,
+      "reward_std": 3.5544650554656982,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.8609914779663086,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 522
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 104.6,
+      "grad_norm": 0.6660361886024475,
+      "kl": 1.1192728281021118,
+      "learning_rate": 4.25615916529213e-06,
+      "loss": 0.0448,
+      "reward": 3.0469722747802734,
+      "reward_std": 0.6083049178123474,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9285714626312256,
+      "rewards/wrapped_driving_reward": 0.36840105056762695,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 523
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 104.8,
+      "grad_norm": 0.5429725646972656,
+      "kl": 1.0266575813293457,
+      "learning_rate": 4.2522731607496275e-06,
+      "loss": 0.0411,
+      "reward": 2.9466092586517334,
+      "reward_std": 0.3607306480407715,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.05339070409536362,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 524
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 105.0,
+      "grad_norm": 0.5302778482437134,
+      "kl": 1.0682640075683594,
+      "learning_rate": 4.248378816008418e-06,
+      "loss": 0.0427,
+      "reward": 3.572202205657959,
+      "reward_std": 0.28779879212379456,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8222021460533142,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 525
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 105.2,
+      "grad_norm": 0.6403073072433472,
+      "kl": 1.3778810501098633,
+      "learning_rate": 4.244476149604201e-06,
+      "loss": 0.0551,
+      "reward": 1.7839674949645996,
+      "reward_std": 3.200443744659424,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.7160324454307556,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 526
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 105.4,
+      "grad_norm": 0.48363983631134033,
+      "kl": 0.6617292761802673,
+      "learning_rate": 4.2405651801122835e-06,
+      "loss": 0.0265,
+      "reward": 0.5352749824523926,
+      "reward_std": 2.7295193672180176,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.5897250175476074,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 527
+    },
+    {
+      "completion_length": 454.0,
+      "epoch": 105.6,
+      "grad_norm": 0.49991506338119507,
+      "kl": 1.0598442554473877,
+      "learning_rate": 4.236645926147493e-06,
+      "loss": 0.0424,
+      "reward": 3.2137718200683594,
+      "reward_std": 0.2735251486301422,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.935606062412262,
+      "rewards/wrapped_driving_reward": 0.40316566824913025,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 528
+    },
+    {
+      "completion_length": 427.0,
+      "epoch": 105.8,
+      "grad_norm": 0.5063905119895935,
+      "kl": 0.9304898977279663,
+      "learning_rate": 4.2327184063640905e-06,
+      "loss": 0.0372,
+      "reward": 2.7854347229003906,
+      "reward_std": 0.10556995123624802,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.949999988079071,
+      "rewards/wrapped_driving_reward": -0.16456523537635803,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 529
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 106.0,
+      "grad_norm": 0.5251207947731018,
+      "kl": 0.7667698264122009,
+      "learning_rate": 4.228782639455674e-06,
+      "loss": 0.0307,
+      "reward": 3.4631171226501465,
+      "reward_std": 0.4566783905029297,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.7131170630455017,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 530
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 106.2,
+      "grad_norm": 0.7626741528511047,
+      "kl": 0.8085213303565979,
+      "learning_rate": 4.224838644155099e-06,
+      "loss": 0.0323,
+      "reward": 2.4170961380004883,
+      "reward_std": 0.5227943062782288,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.20790360867977142,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 531
+    },
+    {
+      "completion_length": 423.0,
+      "epoch": 106.4,
+      "grad_norm": 0.5680715441703796,
+      "kl": 0.6939449310302734,
+      "learning_rate": 4.220886439234385e-06,
+      "loss": 0.0278,
+      "reward": 2.497769355773926,
+      "reward_std": 0.5466521978378296,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.1272306740283966,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 532
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 106.6,
+      "grad_norm": 0.6183933615684509,
+      "kl": 1.1602180004119873,
+      "learning_rate": 4.216926043504626e-06,
+      "loss": 0.0464,
+      "reward": 2.2846357822418213,
+      "reward_std": 2.229444980621338,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.46536412835121155,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 533
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 106.8,
+      "grad_norm": 0.6668643355369568,
+      "kl": 0.7370520234107971,
+      "learning_rate": 4.212957475815898e-06,
+      "loss": 0.0295,
+      "reward": 1.1688251495361328,
+      "reward_std": 3.1426844596862793,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9561749696731567,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 534
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 107.0,
+      "grad_norm": 0.537616491317749,
+      "kl": 1.0950063467025757,
+      "learning_rate": 4.2089807550571786e-06,
+      "loss": 0.0438,
+      "reward": 3.4438555240631104,
+      "reward_std": 0.7739672660827637,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9464285373687744,
+      "rewards/wrapped_driving_reward": 0.7474268674850464,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 535
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 107.2,
+      "grad_norm": 0.625481367111206,
+      "kl": 1.0519458055496216,
+      "learning_rate": 4.204995900156247e-06,
+      "loss": 0.0421,
+      "reward": 2.311418294906616,
+      "reward_std": 2.2375292778015137,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.5635817646980286,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 536
+    },
+    {
+      "completion_length": 465.0,
+      "epoch": 107.4,
+      "grad_norm": 0.46361467242240906,
+      "kl": 0.9292150735855103,
+      "learning_rate": 4.2010029300795986e-06,
+      "loss": 0.0372,
+      "reward": -1.5,
+      "reward_std": 1.0,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 537
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 107.6,
+      "grad_norm": 0.5417262315750122,
+      "kl": 1.2354282140731812,
+      "learning_rate": 4.197001863832355e-06,
+      "loss": 0.0494,
+      "reward": 2.9962191581726074,
+      "reward_std": 0.42118921875953674,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.24621909856796265,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 538
+    },
+    {
+      "completion_length": 403.0,
+      "epoch": 107.8,
+      "grad_norm": 0.5251120924949646,
+      "kl": 0.6911302804946899,
+      "learning_rate": 4.192992720458172e-06,
+      "loss": 0.0276,
+      "reward": 3.7265474796295166,
+      "reward_std": 0.1257975995540619,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.96875,
+      "rewards/wrapped_driving_reward": 0.7577975392341614,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 539
+    },
+    {
+      "completion_length": 484.0,
+      "epoch": 108.0,
+      "grad_norm": 0.4822171628475189,
+      "kl": 0.48192501068115234,
+      "learning_rate": 4.188975519039151e-06,
+      "loss": 0.0193,
+      "reward": 2.7973673343658447,
+      "reward_std": 0.40795889496803284,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.29736727476119995,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 540
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 108.2,
+      "grad_norm": 0.5852164626121521,
+      "kl": 1.5952774286270142,
+      "learning_rate": 4.184950278695745e-06,
+      "loss": 0.0638,
+      "reward": 1.0873538255691528,
+      "reward_std": 3.0677762031555176,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.1626461744308472,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 541
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 108.4,
+      "grad_norm": 0.6261100769042969,
+      "kl": 0.29827645421028137,
+      "learning_rate": 4.18091701858667e-06,
+      "loss": 0.0119,
+      "reward": 1.9164016246795654,
+      "reward_std": 1.2969717979431152,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.7085983753204346,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 542
+    },
+    {
+      "completion_length": 472.0,
+      "epoch": 108.6,
+      "grad_norm": 0.6102639436721802,
+      "kl": 1.2650554180145264,
+      "learning_rate": 4.1768757579088145e-06,
+      "loss": 0.0506,
+      "reward": 3.090078353881836,
+      "reward_std": 0.4966849684715271,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.3400783836841583,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 543
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 108.8,
+      "grad_norm": 0.5978904366493225,
+      "kl": 0.4474976360797882,
+      "learning_rate": 4.172826515897146e-06,
+      "loss": 0.0179,
+      "reward": -1.7957313060760498,
+      "reward_std": 1.3004844188690186,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7272727489471436,
+      "rewards/wrapped_driving_reward": -3.8980040550231934,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 544
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 109.0,
+      "grad_norm": 0.5209200382232666,
+      "kl": 1.0046333074569702,
+      "learning_rate": 4.168769311824619e-06,
+      "loss": 0.0402,
+      "reward": 3.4396934509277344,
+      "reward_std": 0.48305127024650574,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.984375,
+      "rewards/wrapped_driving_reward": 0.8303184509277344,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 545
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 109.2,
+      "grad_norm": 0.6050389409065247,
+      "kl": 1.4569886922836304,
+      "learning_rate": 4.164704165002086e-06,
+      "loss": 0.0583,
+      "reward": 3.0283496379852295,
+      "reward_std": 0.3845747411251068,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.984375,
+      "rewards/wrapped_driving_reward": 0.16897478699684143,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 546
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 109.4,
+      "grad_norm": 0.6651176810264587,
+      "kl": 0.5778868794441223,
+      "learning_rate": 4.160631094778205e-06,
+      "loss": 0.0231,
+      "reward": 2.923994541168213,
+      "reward_std": 0.7468093037605286,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.17399446666240692,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 547
+    },
+    {
+      "completion_length": 479.0,
+      "epoch": 109.6,
+      "grad_norm": 0.5163626670837402,
+      "kl": 0.47668424248695374,
+      "learning_rate": 4.1565501205393445e-06,
+      "loss": 0.0191,
+      "reward": 2.7185633182525635,
+      "reward_std": 0.14974838495254517,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.2814367413520813,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 548
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 109.8,
+      "grad_norm": 0.5523006916046143,
+      "kl": 1.2691748142242432,
+      "learning_rate": 4.152461261709494e-06,
+      "loss": 0.0508,
+      "reward": 3.828211784362793,
+      "reward_std": 0.023511776700615883,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8282119035720825,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 549
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 110.0,
+      "grad_norm": 0.5335929989814758,
+      "kl": 0.9684991240501404,
+      "learning_rate": 4.1483645377501726e-06,
+      "loss": 0.0387,
+      "reward": 3.222689151763916,
+      "reward_std": 0.20909461379051208,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.34768906235694885,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 550
+    },
+    {
+      "completion_length": 460.0,
+      "epoch": 110.2,
+      "grad_norm": 0.5376462936401367,
+      "kl": 0.7713929414749146,
+      "learning_rate": 4.144259968160332e-06,
+      "loss": 0.0309,
+      "reward": 3.4893195629119873,
+      "reward_std": 0.6945598721504211,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.7393194437026978,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 551
+    },
+    {
+      "completion_length": 464.0,
+      "epoch": 110.4,
+      "grad_norm": 0.5207159519195557,
+      "kl": 1.09281587600708,
+      "learning_rate": 4.140147572476269e-06,
+      "loss": 0.0437,
+      "reward": 2.8704469203948975,
+      "reward_std": 0.04285159707069397,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9722222089767456,
+      "rewards/wrapped_driving_reward": -0.10177526623010635,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 552
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 110.6,
+      "grad_norm": 0.5194886326789856,
+      "kl": 0.6254560351371765,
+      "learning_rate": 4.136027370271526e-06,
+      "loss": 0.025,
+      "reward": 2.7038888931274414,
+      "reward_std": 0.6243050694465637,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.046111032366752625,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 553
+    },
+    {
+      "completion_length": 378.0,
+      "epoch": 110.8,
+      "grad_norm": 0.5150373578071594,
+      "kl": 1.3050411939620972,
+      "learning_rate": 4.1318993811568065e-06,
+      "loss": 0.0522,
+      "reward": 2.7843098640441895,
+      "reward_std": 0.2614060640335083,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9642857313156128,
+      "rewards/wrapped_driving_reward": -0.1799759566783905,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 554
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 111.0,
+      "grad_norm": 0.4927043616771698,
+      "kl": 1.0330181121826172,
+      "learning_rate": 4.127763624779873e-06,
+      "loss": 0.0413,
+      "reward": 2.8825972080230713,
+      "reward_std": 0.6047881245613098,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.2575971484184265,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 555
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 111.2,
+      "grad_norm": 0.6897583603858948,
+      "kl": 1.2359685897827148,
+      "learning_rate": 4.123620120825459e-06,
+      "loss": 0.0494,
+      "reward": 3.281771183013916,
+      "reward_std": 0.47440090775489807,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.5317711234092712,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 556
+    },
+    {
+      "completion_length": 495.0,
+      "epoch": 111.4,
+      "grad_norm": 0.5989323854446411,
+      "kl": 1.2655354738235474,
+      "learning_rate": 4.119468889015175e-06,
+      "loss": 0.0506,
+      "reward": 2.762580394744873,
+      "reward_std": 0.01977536454796791,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9750000238418579,
+      "rewards/wrapped_driving_reward": -0.21241959929466248,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 557
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 111.6,
+      "grad_norm": 0.5576749444007874,
+      "kl": 0.46419841051101685,
+      "learning_rate": 4.11530994910741e-06,
+      "loss": 0.0186,
+      "reward": 0.5449908375740051,
+      "reward_std": 3.1576852798461914,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.5800092220306396,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 558
+    },
+    {
+      "completion_length": 424.0,
+      "epoch": 111.8,
+      "grad_norm": 0.5356802940368652,
+      "kl": 0.6746591329574585,
+      "learning_rate": 4.111143320897244e-06,
+      "loss": 0.027,
+      "reward": 2.571215867996216,
+      "reward_std": 0.6723935008049011,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.4287840723991394,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 559
+    },
+    {
+      "completion_length": 390.0,
+      "epoch": 112.0,
+      "grad_norm": 0.6406697630882263,
+      "kl": 0.9746482968330383,
+      "learning_rate": 4.106969024216348e-06,
+      "loss": 0.039,
+      "reward": 2.444140672683716,
+      "reward_std": 2.3113973140716553,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9886363744735718,
+      "rewards/wrapped_driving_reward": -0.5444957613945007,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 560
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 112.2,
+      "grad_norm": 0.48831650614738464,
+      "kl": 1.0716817378997803,
+      "learning_rate": 4.102787078932896e-06,
+      "loss": 0.0429,
+      "reward": 0.15067720413208008,
+      "reward_std": 3.1778695583343506,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7142857313156128,
+      "rewards/wrapped_driving_reward": -1.9386086463928223,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 561
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 112.4,
+      "grad_norm": 0.7172489166259766,
+      "kl": 1.6123234033584595,
+      "learning_rate": 4.098597504951462e-06,
+      "loss": 0.0645,
+      "reward": 1.052835464477539,
+      "reward_std": 3.0422089099884033,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.699999988079071,
+      "rewards/wrapped_driving_reward": -1.1471645832061768,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 562
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 112.6,
+      "grad_norm": 0.4700511693954468,
+      "kl": 1.5025134086608887,
+      "learning_rate": 4.094400322212933e-06,
+      "loss": 0.0601,
+      "reward": 3.1847565174102783,
+      "reward_std": 0.271543949842453,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.18475648760795593,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 563
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 112.8,
+      "grad_norm": 0.5466091632843018,
+      "kl": 1.3995592594146729,
+      "learning_rate": 4.09019555069441e-06,
+      "loss": 0.056,
+      "reward": 3.2351319789886475,
+      "reward_std": 0.10810267180204391,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.3601318895816803,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 564
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 113.0,
+      "grad_norm": 0.47437676787376404,
+      "kl": 0.7883629202842712,
+      "learning_rate": 4.085983210409114e-06,
+      "loss": 0.0315,
+      "reward": 1.85590398311615,
+      "reward_std": 3.577000379562378,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.734375,
+      "rewards/wrapped_driving_reward": -0.3784710764884949,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 565
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 113.2,
+      "grad_norm": 0.5034269690513611,
+      "kl": 1.2129865884780884,
+      "learning_rate": 4.081763321406291e-06,
+      "loss": 0.0485,
+      "reward": 2.7711429595947266,
+      "reward_std": 0.5451120734214783,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.22885683178901672,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 566
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 113.4,
+      "grad_norm": 0.469226598739624,
+      "kl": 0.9821814298629761,
+      "learning_rate": 4.077535903771115e-06,
+      "loss": 0.0393,
+      "reward": 3.4845423698425293,
+      "reward_std": 0.1735163778066635,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9642857313156128,
+      "rewards/wrapped_driving_reward": 0.5202568173408508,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 567
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 113.6,
+      "grad_norm": 0.5355891585350037,
+      "kl": 1.099183440208435,
+      "learning_rate": 4.073300977624594e-06,
+      "loss": 0.044,
+      "reward": 2.550929307937622,
+      "reward_std": 0.5364440679550171,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9772727489471436,
+      "rewards/wrapped_driving_reward": -0.17634353041648865,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 568
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 113.8,
+      "grad_norm": 1.7050570249557495,
+      "kl": 1.0493282079696655,
+      "learning_rate": 4.069058563123476e-06,
+      "loss": 0.042,
+      "reward": -1.254978895187378,
+      "reward_std": 0.2830427885055542,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -3.754978895187378,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 569
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 114.0,
+      "grad_norm": 0.5383164882659912,
+      "kl": 0.8460338115692139,
+      "learning_rate": 4.064808680460149e-06,
+      "loss": 0.0338,
+      "reward": 3.1393980979919434,
+      "reward_std": 0.4082465171813965,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.935606062412262,
+      "rewards/wrapped_driving_reward": 0.45379188656806946,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 570
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 114.2,
+      "grad_norm": 0.48489129543304443,
+      "kl": 0.6688995361328125,
+      "learning_rate": 4.060551349862545e-06,
+      "loss": 0.0268,
+      "reward": 2.5152158737182617,
+      "reward_std": 0.3381154239177704,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.35978400707244873,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 571
+    },
+    {
+      "completion_length": 382.0,
+      "epoch": 114.4,
+      "grad_norm": 0.6301425099372864,
+      "kl": 0.767602801322937,
+      "learning_rate": 4.056286591594049e-06,
+      "loss": 0.0307,
+      "reward": 2.103844165802002,
+      "reward_std": 2.120149612426758,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.7711557745933533,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 572
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 114.6,
+      "grad_norm": 0.5594694018363953,
+      "kl": 0.8042862415313721,
+      "learning_rate": 4.052014425953399e-06,
+      "loss": 0.0322,
+      "reward": 2.261784553527832,
+      "reward_std": 0.5970240235328674,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.2382153868675232,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 573
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 114.8,
+      "grad_norm": 0.5271262526512146,
+      "kl": 0.7885665893554688,
+      "learning_rate": 4.047734873274586e-06,
+      "loss": 0.0315,
+      "reward": 3.6224513053894043,
+      "reward_std": 0.416412889957428,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9166666865348816,
+      "rewards/wrapped_driving_reward": 0.8307844400405884,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 574
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 115.0,
+      "grad_norm": 0.6031838655471802,
+      "kl": 0.8005996346473694,
+      "learning_rate": 4.043447953926763e-06,
+      "loss": 0.032,
+      "reward": 1.5088732242584229,
+      "reward_std": 3.370800018310547,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7272727489471436,
+      "rewards/wrapped_driving_reward": -0.8433995246887207,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 575
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 115.2,
+      "grad_norm": 0.5976753234863281,
+      "kl": 1.0529546737670898,
+      "learning_rate": 4.039153688314146e-06,
+      "loss": 0.0421,
+      "reward": 0.6955030560493469,
+      "reward_std": 3.1456363201141357,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.5833333730697632,
+      "rewards/wrapped_driving_reward": -1.3878302574157715,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 576
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 115.4,
+      "grad_norm": 0.5516257882118225,
+      "kl": 1.106783390045166,
+      "learning_rate": 4.034852096875917e-06,
+      "loss": 0.0443,
+      "reward": 3.089322566986084,
+      "reward_std": 0.561561107635498,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.589322566986084,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 577
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 115.6,
+      "grad_norm": 0.5982745289802551,
+      "kl": 0.8650901913642883,
+      "learning_rate": 4.0305432000861236e-06,
+      "loss": 0.0346,
+      "reward": -0.42720913887023926,
+      "reward_std": 2.9172680377960205,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -2.6772091388702393,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 578
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 115.8,
+      "grad_norm": 0.5181532502174377,
+      "kl": 0.4505057632923126,
+      "learning_rate": 4.026227018453587e-06,
+      "loss": 0.018,
+      "reward": 0.1567600965499878,
+      "reward_std": 4.2224836349487305,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.5932399034500122,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 579
+    },
+    {
+      "completion_length": 477.0,
+      "epoch": 116.0,
+      "grad_norm": 0.5441889762878418,
+      "kl": 1.122840404510498,
+      "learning_rate": 4.021903572521802e-06,
+      "loss": 0.0449,
+      "reward": 2.3626151084899902,
+      "reward_std": 1.115975022315979,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.6373847723007202,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 580
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 116.2,
+      "grad_norm": 0.6262593269348145,
+      "kl": 1.4455444812774658,
+      "learning_rate": 4.0175728828688355e-06,
+      "loss": 0.0578,
+      "reward": 3.0197038650512695,
+      "reward_std": 0.16474168002605438,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.019704071804881096,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 581
+    },
+    {
+      "completion_length": 455.0,
+      "epoch": 116.4,
+      "grad_norm": 0.7343289256095886,
+      "kl": 1.0988672971725464,
+      "learning_rate": 4.013234970107236e-06,
+      "loss": 0.044,
+      "reward": 3.284791946411133,
+      "reward_std": 0.6016852259635925,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9722222089767456,
+      "rewards/wrapped_driving_reward": 0.5625696182250977,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 582
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 116.6,
+      "grad_norm": 0.7269496917724609,
+      "kl": 1.15138578414917,
+      "learning_rate": 4.0088898548839285e-06,
+      "loss": 0.0461,
+      "reward": 1.8786382675170898,
+      "reward_std": 1.9626902341842651,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.8713617324829102,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 583
+    },
+    {
+      "completion_length": 457.0,
+      "epoch": 116.8,
+      "grad_norm": 0.5865412354469299,
+      "kl": 1.0894410610198975,
+      "learning_rate": 4.0045375578801216e-06,
+      "loss": 0.0436,
+      "reward": 1.158536434173584,
+      "reward_std": 2.4944934844970703,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.841463565826416,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 584
+    },
+    {
+      "completion_length": 493.0,
+      "epoch": 117.0,
+      "grad_norm": 0.5936154127120972,
+      "kl": 1.2170442342758179,
+      "learning_rate": 4.000178099811203e-06,
+      "loss": 0.0487,
+      "reward": 3.4582886695861816,
+      "reward_std": 0.7560064792633057,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.4582887887954712,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 585
+    },
+    {
+      "completion_length": 395.0,
+      "epoch": 117.2,
+      "grad_norm": 0.6214985251426697,
+      "kl": 1.1934869289398193,
+      "learning_rate": 3.995811501426648e-06,
+      "loss": 0.0477,
+      "reward": 1.7916817665100098,
+      "reward_std": 1.8864822387695312,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.2083182334899902,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 586
+    },
+    {
+      "completion_length": 422.0,
+      "epoch": 117.4,
+      "grad_norm": 0.5442876815795898,
+      "kl": 0.6532779932022095,
+      "learning_rate": 3.991437783509916e-06,
+      "loss": 0.0261,
+      "reward": 2.1177978515625,
+      "reward_std": 3.411893129348755,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.3822019100189209,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 587
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 117.6,
+      "grad_norm": 0.5017068982124329,
+      "kl": 0.4669530391693115,
+      "learning_rate": 3.987056966878354e-06,
+      "loss": 0.0187,
+      "reward": 2.701904773712158,
+      "reward_std": 0.4848705530166626,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.20190489292144775,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 588
+    },
+    {
+      "completion_length": 451.0,
+      "epoch": 117.8,
+      "grad_norm": 0.6502745151519775,
+      "kl": 0.9627188444137573,
+      "learning_rate": 3.982669072383093e-06,
+      "loss": 0.0385,
+      "reward": 3.4546749591827393,
+      "reward_std": 0.22837932407855988,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9750000238418579,
+      "rewards/wrapped_driving_reward": 0.4796748757362366,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 589
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 118.0,
+      "grad_norm": 0.5828717947006226,
+      "kl": 1.0402276515960693,
+      "learning_rate": 3.978274120908957e-06,
+      "loss": 0.0416,
+      "reward": 1.3493000268936157,
+      "reward_std": 3.240011215209961,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.0256999731063843,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 590
+    },
+    {
+      "completion_length": 441.0,
+      "epoch": 118.2,
+      "grad_norm": 0.4777461588382721,
+      "kl": 0.8824390769004822,
+      "learning_rate": 3.973872133374354e-06,
+      "loss": 0.0353,
+      "reward": 2.3634040355682373,
+      "reward_std": 0.32662835717201233,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.875,
+      "rewards/wrapped_driving_reward": -0.3865959048271179,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 591
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 118.4,
+      "grad_norm": 0.4983525276184082,
+      "kl": 1.1703006029129028,
+      "learning_rate": 3.969463130731183e-06,
+      "loss": 0.0468,
+      "reward": 3.0047945976257324,
+      "reward_std": 0.6505519151687622,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.25479456782341003,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 592
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 118.6,
+      "grad_norm": 0.6996035575866699,
+      "kl": 0.7430159449577332,
+      "learning_rate": 3.965047133964735e-06,
+      "loss": 0.0297,
+      "reward": 1.286207914352417,
+      "reward_std": 3.1933374404907227,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.963792085647583,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 593
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 118.8,
+      "grad_norm": 0.5467674136161804,
+      "kl": 0.8126164674758911,
+      "learning_rate": 3.960624164093587e-06,
+      "loss": 0.0325,
+      "reward": 2.703213691711426,
+      "reward_std": 0.6341179013252258,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9444444179534912,
+      "rewards/wrapped_driving_reward": 0.13376936316490173,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 594
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 119.0,
+      "grad_norm": 0.531501054763794,
+      "kl": 1.0887751579284668,
+      "learning_rate": 3.956194242169506e-06,
+      "loss": 0.0436,
+      "reward": 3.7124338150024414,
+      "reward_std": 0.24506977200508118,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8374338150024414,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 595
+    },
+    {
+      "completion_length": 406.0,
+      "epoch": 119.2,
+      "grad_norm": 0.592689037322998,
+      "kl": 0.8516889214515686,
+      "learning_rate": 3.951757389277349e-06,
+      "loss": 0.0341,
+      "reward": 2.9845433235168457,
+      "reward_std": 0.557920515537262,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.10954323410987854,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 596
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 119.4,
+      "grad_norm": 0.6375570893287659,
+      "kl": 0.32796257734298706,
+      "learning_rate": 3.947313626534965e-06,
+      "loss": 0.0131,
+      "reward": 1.316743016242981,
+      "reward_std": 3.562058448791504,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.703125,
+      "rewards/wrapped_driving_reward": -0.761381983757019,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 597
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 119.6,
+      "grad_norm": 0.5976330637931824,
+      "kl": 0.8567104339599609,
+      "learning_rate": 3.942862975093085e-06,
+      "loss": 0.0343,
+      "reward": 3.3494763374328613,
+      "reward_std": 0.23357340693473816,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.34947648644447327,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 598
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 119.8,
+      "grad_norm": 0.5101990103721619,
+      "kl": 1.0826239585876465,
+      "learning_rate": 3.938405456135231e-06,
+      "loss": 0.0433,
+      "reward": 3.693582057952881,
+      "reward_std": 0.24203148484230042,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8185819983482361,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 599
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 120.0,
+      "grad_norm": 0.5968356132507324,
+      "kl": 0.4376532733440399,
+      "learning_rate": 3.933941090877615e-06,
+      "loss": 0.0175,
+      "reward": -0.602975606918335,
+      "reward_std": 3.3535501956939697,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -2.227975368499756,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 600
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 120.2,
+      "grad_norm": 0.5347983837127686,
+      "kl": 0.8922858834266663,
+      "learning_rate": 3.929469900569031e-06,
+      "loss": 0.0357,
+      "reward": 1.4616684913635254,
+      "reward_std": 2.0025837421417236,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.6633315086364746,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 601
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 120.4,
+      "grad_norm": 0.49703311920166016,
+      "kl": 1.0437543392181396,
+      "learning_rate": 3.924991906490758e-06,
+      "loss": 0.0418,
+      "reward": -0.2095600962638855,
+      "reward_std": 1.9399936199188232,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -2.8345601558685303,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 602
+    },
+    {
+      "completion_length": 498.0,
+      "epoch": 120.6,
+      "grad_norm": 0.4421629011631012,
+      "kl": 0.9316573739051819,
+      "learning_rate": 3.92050712995646e-06,
+      "loss": 0.0373,
+      "reward": 3.238769054412842,
+      "reward_std": 0.17726895213127136,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.238769069314003,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 603
+    },
+    {
+      "completion_length": 362.0,
+      "epoch": 120.8,
+      "grad_norm": 0.6305619478225708,
+      "kl": 0.8384993672370911,
+      "learning_rate": 3.916015592312083e-06,
+      "loss": 0.0335,
+      "reward": 3.710329532623291,
+      "reward_std": 0.2542549967765808,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8353294134140015,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 604
+    },
+    {
+      "completion_length": 410.0,
+      "epoch": 121.0,
+      "grad_norm": 0.5139946341514587,
+      "kl": 1.0644137859344482,
+      "learning_rate": 3.911517314935752e-06,
+      "loss": 0.0426,
+      "reward": 3.2626075744628906,
+      "reward_std": 0.07001760601997375,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.2626074254512787,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 605
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 121.2,
+      "grad_norm": 0.7288184762001038,
+      "kl": 1.8501490354537964,
+      "learning_rate": 3.907012319237672e-06,
+      "loss": 0.074,
+      "reward": 1.4808318614959717,
+      "reward_std": 3.336132526397705,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7272727489471436,
+      "rewards/wrapped_driving_reward": -0.7464408874511719,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 606
+    },
+    {
+      "completion_length": 427.0,
+      "epoch": 121.4,
+      "grad_norm": 0.5688821077346802,
+      "kl": 1.0949598550796509,
+      "learning_rate": 3.902500626660025e-06,
+      "loss": 0.0438,
+      "reward": -0.19872227311134338,
+      "reward_std": 2.0224311351776123,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8500000238418579,
+      "rewards/wrapped_driving_reward": -3.048722267150879,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 607
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 121.6,
+      "grad_norm": 0.5117161273956299,
+      "kl": 0.8083535432815552,
+      "learning_rate": 3.897982258676867e-06,
+      "loss": 0.0323,
+      "reward": 2.9899044036865234,
+      "reward_std": 0.9233556389808655,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.7291666865348816,
+      "rewards/wrapped_driving_reward": 0.7607378959655762,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 608
+    },
+    {
+      "completion_length": 461.0,
+      "epoch": 121.8,
+      "grad_norm": 0.41981130838394165,
+      "kl": 1.5435020923614502,
+      "learning_rate": 3.8934572367940285e-06,
+      "loss": 0.0617,
+      "reward": 2.585515260696411,
+      "reward_std": 0.15323100984096527,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.41448473930358887,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 609
+    },
+    {
+      "completion_length": 456.0,
+      "epoch": 122.0,
+      "grad_norm": 0.5537935495376587,
+      "kl": 0.9971626996994019,
+      "learning_rate": 3.888925582549006e-06,
+      "loss": 0.0399,
+      "reward": 2.5353288650512695,
+      "reward_std": 2.358036756515503,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9722222089767456,
+      "rewards/wrapped_driving_reward": -0.4368933141231537,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 610
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 122.2,
+      "grad_norm": 0.6289500594139099,
+      "kl": 1.1899405717849731,
+      "learning_rate": 3.8843873175108685e-06,
+      "loss": 0.0476,
+      "reward": 3.100329875946045,
+      "reward_std": 0.530876874923706,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.96875,
+      "rewards/wrapped_driving_reward": 0.38157975673675537,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 611
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 122.4,
+      "grad_norm": 0.533795952796936,
+      "kl": 1.278198003768921,
+      "learning_rate": 3.879842463280146e-06,
+      "loss": 0.0511,
+      "reward": 2.8987560272216797,
+      "reward_std": 0.5386033058166504,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.148756206035614,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 612
+    },
+    {
+      "completion_length": 360.0,
+      "epoch": 122.6,
+      "grad_norm": 0.5524008274078369,
+      "kl": 0.7699089050292969,
+      "learning_rate": 3.875291041488734e-06,
+      "loss": 0.0308,
+      "reward": 3.1413626670837402,
+      "reward_std": 0.4477062523365021,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9230769276618958,
+      "rewards/wrapped_driving_reward": 0.7182857990264893,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 613
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 122.8,
+      "grad_norm": 0.4463861882686615,
+      "kl": 1.3604273796081543,
+      "learning_rate": 3.870733073799785e-06,
+      "loss": 0.0544,
+      "reward": 2.821460247039795,
+      "reward_std": 0.21004045009613037,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9375,
+      "rewards/wrapped_driving_reward": -0.11603959649801254,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 614
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 123.0,
+      "grad_norm": 0.5090853571891785,
+      "kl": 0.8209176063537598,
+      "learning_rate": 3.866168581907609e-06,
+      "loss": 0.0328,
+      "reward": 3.033195734024048,
+      "reward_std": 0.2945842444896698,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.1581955850124359,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 615
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 123.2,
+      "grad_norm": 0.5075961947441101,
+      "kl": 0.6219139695167542,
+      "learning_rate": 3.861597587537568e-06,
+      "loss": 0.0249,
+      "reward": -0.6715507507324219,
+      "reward_std": 2.605349063873291,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -3.046550750732422,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 616
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 123.4,
+      "grad_norm": 0.6667484045028687,
+      "kl": 1.1634588241577148,
+      "learning_rate": 3.8570201124459745e-06,
+      "loss": 0.0465,
+      "reward": 1.6204893589019775,
+      "reward_std": 3.7763609886169434,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.37951064109802246,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 617
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 123.6,
+      "grad_norm": 0.6611601114273071,
+      "kl": 0.5242982506752014,
+      "learning_rate": 3.8524361784199855e-06,
+      "loss": 0.021,
+      "reward": 2.819260358810425,
+      "reward_std": 0.5874485373497009,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.3192603886127472,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 618
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 123.8,
+      "grad_norm": 1.5914947986602783,
+      "kl": 1.2179936170578003,
+      "learning_rate": 3.847845807277501e-06,
+      "loss": 0.0487,
+      "reward": 1.4335602521896362,
+      "reward_std": 2.353290319442749,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.0664397478103638,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 619
+    },
+    {
+      "completion_length": 462.0,
+      "epoch": 124.0,
+      "grad_norm": 0.6235033273696899,
+      "kl": 0.9362177848815918,
+      "learning_rate": 3.8432490208670605e-06,
+      "loss": 0.0374,
+      "reward": 2.7716119289398193,
+      "reward_std": 0.6671396493911743,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9166666865348816,
+      "rewards/wrapped_driving_reward": -0.14505480229854584,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 620
+    },
+    {
+      "completion_length": 347.0,
+      "epoch": 124.2,
+      "grad_norm": 0.824704110622406,
+      "kl": 0.8133590221405029,
+      "learning_rate": 3.838645841067735e-06,
+      "loss": 0.0325,
+      "reward": 1.1893500089645386,
+      "reward_std": 2.809638023376465,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.5833333730697632,
+      "rewards/wrapped_driving_reward": -1.1439833641052246,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 621
+    },
+    {
+      "completion_length": 386.0,
+      "epoch": 124.4,
+      "grad_norm": 0.7511218190193176,
+      "kl": 1.0320838689804077,
+      "learning_rate": 3.83403628978903e-06,
+      "loss": 0.0413,
+      "reward": 1.5405443906784058,
+      "reward_std": 3.0421595573425293,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9594556093215942,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 622
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 124.6,
+      "grad_norm": 0.7474787831306458,
+      "kl": 2.0438644886016846,
+      "learning_rate": 3.829420388970772e-06,
+      "loss": 0.0818,
+      "reward": 3.1411290168762207,
+      "reward_std": 0.30362197756767273,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.3911292254924774,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 623
+    },
+    {
+      "completion_length": 489.0,
+      "epoch": 124.8,
+      "grad_norm": 0.5865500569343567,
+      "kl": 1.1531330347061157,
+      "learning_rate": 3.824798160583012e-06,
+      "loss": 0.0461,
+      "reward": 1.3933429718017578,
+      "reward_std": 1.692426323890686,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.3566570281982422,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 624
+    },
+    {
+      "completion_length": 497.0,
+      "epoch": 125.0,
+      "grad_norm": 1.8625023365020752,
+      "kl": 1.2559576034545898,
+      "learning_rate": 3.82016962662592e-06,
+      "loss": 0.0502,
+      "reward": 3.264035224914551,
+      "reward_std": 0.47378936409950256,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.26403525471687317,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 625
+    },
+    {
+      "completion_length": 424.0,
+      "epoch": 125.2,
+      "grad_norm": 0.5372361540794373,
+      "kl": 1.0564872026443481,
+      "learning_rate": 3.815534809129674e-06,
+      "loss": 0.0423,
+      "reward": 3.686570167541504,
+      "reward_std": 0.2549721598625183,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8115702271461487,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 626
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 125.4,
+      "grad_norm": 0.6623659133911133,
+      "kl": 1.3536056280136108,
+      "learning_rate": 3.8108937301543613e-06,
+      "loss": 0.0541,
+      "reward": 1.6809741258621216,
+      "reward_std": 3.1360626220703125,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.6940257549285889,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 627
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 125.6,
+      "grad_norm": 0.7702702879905701,
+      "kl": 0.5843633413314819,
+      "learning_rate": 3.806246411789872e-06,
+      "loss": 0.0234,
+      "reward": 0.16281354427337646,
+      "reward_std": 2.0939905643463135,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -2.337186336517334,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 628
+    },
+    {
+      "completion_length": 451.0,
+      "epoch": 125.8,
+      "grad_norm": 0.567046046257019,
+      "kl": 0.6312512755393982,
+      "learning_rate": 3.8015928761557937e-06,
+      "loss": 0.0253,
+      "reward": 0.7392481565475464,
+      "reward_std": 2.621385335922241,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.8857518434524536,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 629
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 126.0,
+      "grad_norm": 0.5259938836097717,
+      "kl": 1.050511121749878,
+      "learning_rate": 3.796933145401304e-06,
+      "loss": 0.042,
+      "reward": 1.5790135860443115,
+      "reward_std": 3.3867099285125732,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.7959863543510437,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 630
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 126.2,
+      "grad_norm": 0.5427968502044678,
+      "kl": 1.2725714445114136,
+      "learning_rate": 3.7922672417050687e-06,
+      "loss": 0.0509,
+      "reward": 2.798665761947632,
+      "reward_std": 0.6321271657943726,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.875,
+      "rewards/wrapped_driving_reward": 0.2986658215522766,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 631
+    },
+    {
+      "completion_length": 437.0,
+      "epoch": 126.4,
+      "grad_norm": 0.5535632371902466,
+      "kl": 1.0805319547653198,
+      "learning_rate": 3.787595187275136e-06,
+      "loss": 0.0432,
+      "reward": 3.469416379928589,
+      "reward_std": 0.41116681694984436,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9375,
+      "rewards/wrapped_driving_reward": 0.6569163799285889,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 632
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 126.6,
+      "grad_norm": 1.1283907890319824,
+      "kl": 1.4552005529403687,
+      "learning_rate": 3.782917004348826e-06,
+      "loss": 0.0582,
+      "reward": 2.5764386653900146,
+      "reward_std": 0.25892573595046997,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8999999761581421,
+      "rewards/wrapped_driving_reward": -0.19856137037277222,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 633
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 126.8,
+      "grad_norm": 0.6342353224754333,
+      "kl": 1.0458495616912842,
+      "learning_rate": 3.77823271519263e-06,
+      "loss": 0.0418,
+      "reward": 1.4458222389221191,
+      "reward_std": 2.164912223815918,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.1791776418685913,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 634
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 127.0,
+      "grad_norm": 0.5013647079467773,
+      "kl": 1.0987770557403564,
+      "learning_rate": 3.773542342102105e-06,
+      "loss": 0.044,
+      "reward": 2.5572562217712402,
+      "reward_std": 0.5937471985816956,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9583333134651184,
+      "rewards/wrapped_driving_reward": 0.09892302751541138,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 635
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 127.2,
+      "grad_norm": 0.4656875729560852,
+      "kl": 1.0471619367599487,
+      "learning_rate": 3.768845907401761e-06,
+      "loss": 0.0419,
+      "reward": 2.2354865074157715,
+      "reward_std": 0.9342080950737,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.6395134329795837,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 636
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 127.4,
+      "grad_norm": 0.575062096118927,
+      "kl": 1.259049654006958,
+      "learning_rate": 3.764143433444962e-06,
+      "loss": 0.0504,
+      "reward": 2.576545000076294,
+      "reward_std": 0.6139914989471436,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9375,
+      "rewards/wrapped_driving_reward": 0.014045089483261108,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 637
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 127.6,
+      "grad_norm": 0.525324821472168,
+      "kl": 0.8727381229400635,
+      "learning_rate": 3.759434942613816e-06,
+      "loss": 0.0349,
+      "reward": 1.5306200981140137,
+      "reward_std": 3.3567087650299072,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.7193797826766968,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 638
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 127.8,
+      "grad_norm": 0.4832088053226471,
+      "kl": 0.7729612588882446,
+      "learning_rate": 3.75472045731907e-06,
+      "loss": 0.0309,
+      "reward": 2.4255733489990234,
+      "reward_std": 0.7332895398139954,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.32442671060562134,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 639
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 128.0,
+      "grad_norm": 0.5773348808288574,
+      "kl": 1.278677225112915,
+      "learning_rate": 3.7500000000000005e-06,
+      "loss": 0.0511,
+      "reward": 1.4694275856018066,
+      "reward_std": 2.9984025955200195,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.6555723547935486,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 640
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 128.2,
+      "grad_norm": 0.6425707936286926,
+      "kl": 0.8203465342521667,
+      "learning_rate": 3.7452735931243108e-06,
+      "loss": 0.0328,
+      "reward": 2.112128734588623,
+      "reward_std": 0.4703961908817291,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.5128712058067322,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 641
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 128.4,
+      "grad_norm": 0.4806380867958069,
+      "kl": 1.1360549926757812,
+      "learning_rate": 3.7405412591880213e-06,
+      "loss": 0.0454,
+      "reward": 0.18416738510131836,
+      "reward_std": 3.472820520401001,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.6908326148986816,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 642
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 128.6,
+      "grad_norm": 0.5253452062606812,
+      "kl": 0.8624851107597351,
+      "learning_rate": 3.735803020715362e-06,
+      "loss": 0.0345,
+      "reward": 1.8811970949172974,
+      "reward_std": 3.594886302947998,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.36880284547805786,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 643
+    },
+    {
+      "completion_length": 364.0,
+      "epoch": 128.8,
+      "grad_norm": 0.5939153432846069,
+      "kl": 0.9492148160934448,
+      "learning_rate": 3.7310589002586683e-06,
+      "loss": 0.038,
+      "reward": 0.8848087787628174,
+      "reward_std": 2.2207157611846924,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.9901912212371826,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 644
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 129.0,
+      "grad_norm": 0.484019011259079,
+      "kl": 1.1680047512054443,
+      "learning_rate": 3.7263089203982698e-06,
+      "loss": 0.0467,
+      "reward": 1.8454086780548096,
+      "reward_std": 1.9383881092071533,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.7795912027359009,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 645
+    },
+    {
+      "completion_length": 494.0,
+      "epoch": 129.2,
+      "grad_norm": 0.46752461791038513,
+      "kl": 0.7998576164245605,
+      "learning_rate": 3.721553103742388e-06,
+      "loss": 0.032,
+      "reward": 2.1568634510040283,
+      "reward_std": 2.207801103591919,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9791666865348816,
+      "rewards/wrapped_driving_reward": -0.44730305671691895,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 646
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 129.4,
+      "grad_norm": 0.5095931887626648,
+      "kl": 1.2253186702728271,
+      "learning_rate": 3.7167914729270205e-06,
+      "loss": 0.049,
+      "reward": 3.187159299850464,
+      "reward_std": 0.5010700225830078,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9886363744735718,
+      "rewards/wrapped_driving_reward": 0.19852286577224731,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 647
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 129.6,
+      "grad_norm": 0.5528401732444763,
+      "kl": 1.5397626161575317,
+      "learning_rate": 3.7120240506158433e-06,
+      "loss": 0.0616,
+      "reward": 3.4808921813964844,
+      "reward_std": 0.575390100479126,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.6058919429779053,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 648
+    },
+    {
+      "completion_length": 487.0,
+      "epoch": 129.8,
+      "grad_norm": 0.5192949771881104,
+      "kl": 1.3175593614578247,
+      "learning_rate": 3.7072508595000935e-06,
+      "loss": 0.0527,
+      "reward": 3.1154708862304688,
+      "reward_std": 0.7258886098861694,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.24047090113162994,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 649
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 130.0,
+      "grad_norm": 0.5336673855781555,
+      "kl": 1.6302653551101685,
+      "learning_rate": 3.7024719222984696e-06,
+      "loss": 0.0652,
+      "reward": 1.334566593170166,
+      "reward_std": 3.2289183139801025,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.040433406829834,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 650
+    },
+    {
+      "completion_length": 486.0,
+      "epoch": 130.2,
+      "grad_norm": 0.7520384192466736,
+      "kl": 1.874834656715393,
+      "learning_rate": 3.6976872617570163e-06,
+      "loss": 0.075,
+      "reward": 3.4488072395324707,
+      "reward_std": 0.258309543132782,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9642857313156128,
+      "rewards/wrapped_driving_reward": 0.6095216274261475,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 651
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 130.4,
+      "grad_norm": 0.7811522483825684,
+      "kl": 0.6162184476852417,
+      "learning_rate": 3.6928969006490212e-06,
+      "loss": 0.0246,
+      "reward": -2.375,
+      "reward_std": 1.6007810831069946,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 652
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 130.6,
+      "grad_norm": 0.4578838050365448,
+      "kl": 1.285402536392212,
+      "learning_rate": 3.6881008617749042e-06,
+      "loss": 0.0514,
+      "reward": 0.9785336256027222,
+      "reward_std": 2.6833934783935547,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.1464663743972778,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 653
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 130.8,
+      "grad_norm": 0.48609647154808044,
+      "kl": 1.111726999282837,
+      "learning_rate": 3.6832991679621087e-06,
+      "loss": 0.0445,
+      "reward": 2.5674822330474854,
+      "reward_std": 0.3593307435512543,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.18251775205135345,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 654
+    },
+    {
+      "completion_length": 455.0,
+      "epoch": 131.0,
+      "grad_norm": 0.5047434568405151,
+      "kl": 1.223645806312561,
+      "learning_rate": 3.6784918420649952e-06,
+      "loss": 0.0489,
+      "reward": 3.3820629119873047,
+      "reward_std": 0.4785292148590088,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9642857313156128,
+      "rewards/wrapped_driving_reward": 0.41777700185775757,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 655
+    },
+    {
+      "completion_length": 433.0,
+      "epoch": 131.2,
+      "grad_norm": 0.5695823431015015,
+      "kl": 1.3213516473770142,
+      "learning_rate": 3.6736789069647273e-06,
+      "loss": 0.0529,
+      "reward": 3.4408211708068848,
+      "reward_std": 0.48655256628990173,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9545454382896423,
+      "rewards/wrapped_driving_reward": 0.4862755835056305,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 656
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 131.4,
+      "grad_norm": 0.5650737881660461,
+      "kl": 0.753746509552002,
+      "learning_rate": 3.6688603855691713e-06,
+      "loss": 0.0301,
+      "reward": 0.73109370470047,
+      "reward_std": 2.6236155033111572,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -2.018906354904175,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 657
+    },
+    {
+      "completion_length": 417.0,
+      "epoch": 131.6,
+      "grad_norm": 0.5839191675186157,
+      "kl": 1.2604615688323975,
+      "learning_rate": 3.664036300812779e-06,
+      "loss": 0.0504,
+      "reward": 3.5584030151367188,
+      "reward_std": 0.49135151505470276,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9750000238418579,
+      "rewards/wrapped_driving_reward": 0.8334031701087952,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 658
+    },
+    {
+      "completion_length": 459.0,
+      "epoch": 131.8,
+      "grad_norm": 0.5347810983657837,
+      "kl": 1.0865789651870728,
+      "learning_rate": 3.6592066756564825e-06,
+      "loss": 0.0435,
+      "reward": 2.519155263900757,
+      "reward_std": 0.6673591732978821,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.10584461688995361,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 659
+    },
+    {
+      "completion_length": 442.0,
+      "epoch": 132.0,
+      "grad_norm": 0.52985680103302,
+      "kl": 0.3356545865535736,
+      "learning_rate": 3.654371533087586e-06,
+      "loss": 0.0134,
+      "reward": 2.64139986038208,
+      "reward_std": 0.9725221395492554,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.016399994492530823,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 660
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 132.2,
+      "grad_norm": 0.48469388484954834,
+      "kl": 1.271719217300415,
+      "learning_rate": 3.64953089611965e-06,
+      "loss": 0.0509,
+      "reward": 2.7419373989105225,
+      "reward_std": 0.08097031712532043,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9642857313156128,
+      "rewards/wrapped_driving_reward": -0.22234830260276794,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 661
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 132.4,
+      "grad_norm": 0.87370365858078,
+      "kl": 1.4457768201828003,
+      "learning_rate": 3.6446847877923917e-06,
+      "loss": 0.0578,
+      "reward": 2.3081374168395996,
+      "reward_std": 2.2228269577026367,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.44186270236968994,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 662
+    },
+    {
+      "completion_length": 410.0,
+      "epoch": 132.6,
+      "grad_norm": 0.5732893943786621,
+      "kl": 1.3890821933746338,
+      "learning_rate": 3.639833231171569e-06,
+      "loss": 0.0556,
+      "reward": 3.4063034057617188,
+      "reward_std": 0.5635956525802612,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9772727489471436,
+      "rewards/wrapped_driving_reward": 0.55403071641922,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 663
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 132.8,
+      "grad_norm": 0.5441069602966309,
+      "kl": 0.8945682048797607,
+      "learning_rate": 3.634976249348867e-06,
+      "loss": 0.0358,
+      "reward": 3.1547274589538574,
+      "reward_std": 0.16029714047908783,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.279727578163147,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 664
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 133.0,
+      "grad_norm": 0.5390021800994873,
+      "kl": 0.566189169883728,
+      "learning_rate": 3.6301138654418e-06,
+      "loss": 0.0226,
+      "reward": 1.5111743211746216,
+      "reward_std": 3.3476204872131348,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.8638256788253784,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 665
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 133.2,
+      "grad_norm": 0.49394407868385315,
+      "kl": 0.46724024415016174,
+      "learning_rate": 3.625246102593588e-06,
+      "loss": 0.0187,
+      "reward": 1.7283051013946533,
+      "reward_std": 3.516357660293579,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.3966948688030243,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 666
+    },
+    {
+      "completion_length": 481.0,
+      "epoch": 133.4,
+      "grad_norm": 0.5267409682273865,
+      "kl": 0.5396498441696167,
+      "learning_rate": 3.6203729839730567e-06,
+      "loss": 0.0216,
+      "reward": 1.739487886428833,
+      "reward_std": 1.8327277898788452,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9750000238418579,
+      "rewards/wrapped_driving_reward": -1.1105120182037354,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 667
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 133.6,
+      "grad_norm": 0.5509145855903625,
+      "kl": 0.605006992816925,
+      "learning_rate": 3.6154945327745223e-06,
+      "loss": 0.0242,
+      "reward": -0.9433751106262207,
+      "reward_std": 2.5427985191345215,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -3.1933751106262207,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 668
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 133.8,
+      "grad_norm": 0.601017415523529,
+      "kl": 0.7718148231506348,
+      "learning_rate": 3.610610772217682e-06,
+      "loss": 0.0309,
+      "reward": 2.710988998413086,
+      "reward_std": 0.43899405002593994,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.28901103138923645,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 669
+    },
+    {
+      "completion_length": 483.0,
+      "epoch": 134.0,
+      "grad_norm": 0.5350854992866516,
+      "kl": 0.619436502456665,
+      "learning_rate": 3.6057217255475034e-06,
+      "loss": 0.0248,
+      "reward": 3.0488271713256836,
+      "reward_std": 0.5188868641853333,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.4238271713256836,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 670
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 134.2,
+      "grad_norm": 0.4458288848400116,
+      "kl": 0.7321600317955017,
+      "learning_rate": 3.600827416034115e-06,
+      "loss": 0.0293,
+      "reward": -0.6971749067306519,
+      "reward_std": 2.9557926654815674,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.45454543828964233,
+      "rewards/wrapped_driving_reward": -2.5267202854156494,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 671
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 134.4,
+      "grad_norm": 0.5819605588912964,
+      "kl": 1.0092272758483887,
+      "learning_rate": 3.595927866972694e-06,
+      "loss": 0.0404,
+      "reward": 3.0731568336486816,
+      "reward_std": 0.34458592534065247,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.1981567144393921,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 672
+    },
+    {
+      "completion_length": 471.0,
+      "epoch": 134.6,
+      "grad_norm": 0.4470769464969635,
+      "kl": 1.3916444778442383,
+      "learning_rate": 3.591023101683355e-06,
+      "loss": 0.0557,
+      "reward": 2.623095750808716,
+      "reward_std": 0.15428505837917328,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.37690430879592896,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 673
+    },
+    {
+      "completion_length": 471.0,
+      "epoch": 134.8,
+      "grad_norm": 0.8402154445648193,
+      "kl": 0.894091010093689,
+      "learning_rate": 3.586113143511043e-06,
+      "loss": 0.0358,
+      "reward": 3.297544479370117,
+      "reward_std": 0.5369318723678589,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9375,
+      "rewards/wrapped_driving_reward": 0.610044538974762,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 674
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 135.0,
+      "grad_norm": 0.6311519742012024,
+      "kl": 0.7051270008087158,
+      "learning_rate": 3.5811980158254156e-06,
+      "loss": 0.0282,
+      "reward": 1.474664330482483,
+      "reward_std": 3.348062515258789,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9003356099128723,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 675
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 135.2,
+      "grad_norm": 0.6317039728164673,
+      "kl": 0.52884840965271,
+      "learning_rate": 3.5762777420207382e-06,
+      "loss": 0.0212,
+      "reward": -0.5374802947044373,
+      "reward_std": 3.7230663299560547,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -2.162480354309082,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 676
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 135.4,
+      "grad_norm": 0.7059259414672852,
+      "kl": 1.1256921291351318,
+      "learning_rate": 3.5713523455157686e-06,
+      "loss": 0.045,
+      "reward": 1.6178922653198242,
+      "reward_std": 3.7729244232177734,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.38210779428482056,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 677
+    },
+    {
+      "completion_length": 469.0,
+      "epoch": 135.6,
+      "grad_norm": 0.699884831905365,
+      "kl": 0.8976281881332397,
+      "learning_rate": 3.566421849753646e-06,
+      "loss": 0.0359,
+      "reward": 2.338595151901245,
+      "reward_std": 0.5884451270103455,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9375,
+      "rewards/wrapped_driving_reward": -0.2239048033952713,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 678
+    },
+    {
+      "completion_length": 493.0,
+      "epoch": 135.8,
+      "grad_norm": 0.8474484086036682,
+      "kl": 1.239915370941162,
+      "learning_rate": 3.5614862782017833e-06,
+      "loss": 0.0496,
+      "reward": 3.6089816093444824,
+      "reward_std": 0.34150734543800354,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.6089816093444824,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 679
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 136.0,
+      "grad_norm": 0.7421549558639526,
+      "kl": 1.3694592714309692,
+      "learning_rate": 3.556545654351749e-06,
+      "loss": 0.0548,
+      "reward": 0.45889168977737427,
+      "reward_std": 2.6780941486358643,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.6041666865348816,
+      "rewards/wrapped_driving_reward": -1.3952749967575073,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 680
+    },
+    {
+      "completion_length": 495.0,
+      "epoch": 136.2,
+      "grad_norm": 0.5081685781478882,
+      "kl": 1.3575365543365479,
+      "learning_rate": 3.551600001719161e-06,
+      "loss": 0.0543,
+      "reward": 3.8419742584228516,
+      "reward_std": 0.012562450021505356,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.841974139213562,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 681
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 136.4,
+      "grad_norm": 0.7341115474700928,
+      "kl": 1.159175992012024,
+      "learning_rate": 3.5466493438435707e-06,
+      "loss": 0.0464,
+      "reward": 1.3301422595977783,
+      "reward_std": 3.229234457015991,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.7948578000068665,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 682
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 136.6,
+      "grad_norm": 0.7937794327735901,
+      "kl": 1.867870807647705,
+      "learning_rate": 3.541693704288355e-06,
+      "loss": 0.0747,
+      "reward": 3.3098440170288086,
+      "reward_std": 0.45399734377861023,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.5598439574241638,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 683
+    },
+    {
+      "completion_length": 440.0,
+      "epoch": 136.8,
+      "grad_norm": 0.540158748626709,
+      "kl": 0.5177903175354004,
+      "learning_rate": 3.536733106640598e-06,
+      "loss": 0.0207,
+      "reward": 1.2544230222702026,
+      "reward_std": 2.3613858222961426,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.7455769777297974,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 684
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 137.0,
+      "grad_norm": 0.45787709951400757,
+      "kl": 0.9340965747833252,
+      "learning_rate": 3.531767574510987e-06,
+      "loss": 0.0374,
+      "reward": 2.6123316287994385,
+      "reward_std": 0.17315728962421417,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9722222089767456,
+      "rewards/wrapped_driving_reward": -0.35989055037498474,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 685
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 137.2,
+      "grad_norm": 0.5473198294639587,
+      "kl": 1.2692763805389404,
+      "learning_rate": 3.5267971315336936e-06,
+      "loss": 0.0508,
+      "reward": 1.327620029449463,
+      "reward_std": 3.2223923206329346,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.047379970550537,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 686
+    },
+    {
+      "completion_length": 449.0,
+      "epoch": 137.4,
+      "grad_norm": 0.7179501056671143,
+      "kl": 0.834479570388794,
+      "learning_rate": 3.5218218013662626e-06,
+      "loss": 0.0334,
+      "reward": 2.7406983375549316,
+      "reward_std": 0.6632347702980042,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9642857313156128,
+      "rewards/wrapped_driving_reward": 0.02641259878873825,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 687
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 137.6,
+      "grad_norm": 0.5952706933021545,
+      "kl": 0.749407172203064,
+      "learning_rate": 3.516841607689501e-06,
+      "loss": 0.03,
+      "reward": 3.403985023498535,
+      "reward_std": 0.48997849225997925,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.403984934091568,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 688
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 137.8,
+      "grad_norm": 0.48037606477737427,
+      "kl": 0.7486713528633118,
+      "learning_rate": 3.511856574207364e-06,
+      "loss": 0.0299,
+      "reward": 3.3666038513183594,
+      "reward_std": 0.11598622053861618,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.36660394072532654,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 689
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 138.0,
+      "grad_norm": 0.6659343242645264,
+      "kl": 0.98530113697052,
+      "learning_rate": 3.5068667246468437e-06,
+      "loss": 0.0394,
+      "reward": 1.5401697158813477,
+      "reward_std": 3.7069664001464844,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7045454382896423,
+      "rewards/wrapped_driving_reward": -0.6643756628036499,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 690
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 138.2,
+      "grad_norm": 0.5222293734550476,
+      "kl": 0.5954757928848267,
+      "learning_rate": 3.5018720827578523e-06,
+      "loss": 0.0238,
+      "reward": 2.6262009143829346,
+      "reward_std": 0.13065750896930695,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.12379903346300125,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 691
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 138.4,
+      "grad_norm": 0.641049325466156,
+      "kl": 1.211357593536377,
+      "learning_rate": 3.496872672313116e-06,
+      "loss": 0.0485,
+      "reward": 1.5172151327133179,
+      "reward_std": 3.099297523498535,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.7327848672866821,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 692
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 138.6,
+      "grad_norm": 0.5524839162826538,
+      "kl": 1.0793451070785522,
+      "learning_rate": 3.491868517108053e-06,
+      "loss": 0.0432,
+      "reward": 2.820830821990967,
+      "reward_std": 0.561494767665863,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.17916926741600037,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 693
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 138.8,
+      "grad_norm": 0.41166210174560547,
+      "kl": 1.6346538066864014,
+      "learning_rate": 3.486859640960668e-06,
+      "loss": 0.0654,
+      "reward": 2.7948412895202637,
+      "reward_std": 0.020073924213647842,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.20515868067741394,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 694
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 139.0,
+      "grad_norm": 0.5615822076797485,
+      "kl": 0.5488652586936951,
+      "learning_rate": 3.481846067711436e-06,
+      "loss": 0.022,
+      "reward": 3.8224549293518066,
+      "reward_std": 0.021474715322256088,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8224549293518066,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 695
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 139.2,
+      "grad_norm": 0.5782674551010132,
+      "kl": 1.0001182556152344,
+      "learning_rate": 3.476827821223184e-06,
+      "loss": 0.04,
+      "reward": 1.3818203210830688,
+      "reward_std": 3.267575263977051,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9931796789169312,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 696
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 139.4,
+      "grad_norm": 0.5270988941192627,
+      "kl": 0.6028380393981934,
+      "learning_rate": 3.4718049253809894e-06,
+      "loss": 0.0241,
+      "reward": 3.23805570602417,
+      "reward_std": 0.320840448141098,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9583333134651184,
+      "rewards/wrapped_driving_reward": 0.40472227334976196,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 697
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 139.6,
+      "grad_norm": 0.4955165982246399,
+      "kl": 0.2747107148170471,
+      "learning_rate": 3.466777404092052e-06,
+      "loss": 0.011,
+      "reward": 1.2135114669799805,
+      "reward_std": 3.2385220527648926,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.03648841381073,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 698
+    },
+    {
+      "completion_length": 435.0,
+      "epoch": 139.8,
+      "grad_norm": 0.580830991268158,
+      "kl": 0.769486665725708,
+      "learning_rate": 3.4617452812855908e-06,
+      "loss": 0.0308,
+      "reward": 3.6568925380706787,
+      "reward_std": 0.10732667148113251,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.824999988079071,
+      "rewards/wrapped_driving_reward": 0.8318926095962524,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 699
+    },
+    {
+      "completion_length": 494.0,
+      "epoch": 140.0,
+      "grad_norm": 0.5295502543449402,
+      "kl": 0.8366852402687073,
+      "learning_rate": 3.4567085809127247e-06,
+      "loss": 0.0335,
+      "reward": 3.1698110103607178,
+      "reward_std": 0.26023775339126587,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.16981105506420135,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 700
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 140.2,
+      "grad_norm": 0.7031523585319519,
+      "kl": 1.2178704738616943,
+      "learning_rate": 3.4516673269463617e-06,
+      "loss": 0.0487,
+      "reward": 3.5846829414367676,
+      "reward_std": 0.5065321922302246,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8346830010414124,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 701
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 140.4,
+      "grad_norm": 0.5828245282173157,
+      "kl": 1.1058602333068848,
+      "learning_rate": 3.4466215433810827e-06,
+      "loss": 0.0442,
+      "reward": 1.7846969366073608,
+      "reward_std": 3.545959949493408,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.5903029441833496,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 702
+    },
+    {
+      "completion_length": 437.0,
+      "epoch": 140.6,
+      "grad_norm": 0.5404889583587646,
+      "kl": 1.335483193397522,
+      "learning_rate": 3.441571254233027e-06,
+      "loss": 0.0534,
+      "reward": 2.553800582885742,
+      "reward_std": 0.27431902289390564,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.1961991786956787,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 703
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 140.8,
+      "grad_norm": 0.4963621497154236,
+      "kl": 0.47123226523399353,
+      "learning_rate": 3.436516483539781e-06,
+      "loss": 0.0188,
+      "reward": 2.3996798992156982,
+      "reward_std": 1.5315988063812256,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9375,
+      "rewards/wrapped_driving_reward": -0.5378199815750122,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 704
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 141.0,
+      "grad_norm": 0.5385916233062744,
+      "kl": 1.076686978340149,
+      "learning_rate": 3.4314572553602577e-06,
+      "loss": 0.0431,
+      "reward": -1.0651588439941406,
+      "reward_std": 2.3442935943603516,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -3.0651588439941406,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 705
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 141.2,
+      "grad_norm": 0.6693921685218811,
+      "kl": 1.052538275718689,
+      "learning_rate": 3.426393593774591e-06,
+      "loss": 0.0421,
+      "reward": 2.958894968032837,
+      "reward_std": 0.5083202123641968,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9722222089767456,
+      "rewards/wrapped_driving_reward": 0.36167269945144653,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 706
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 141.4,
+      "grad_norm": 0.5258262157440186,
+      "kl": 0.5887873768806458,
+      "learning_rate": 3.421325522884013e-06,
+      "loss": 0.0236,
+      "reward": 1.8505744934082031,
+      "reward_std": 3.574678421020508,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.39942556619644165,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 707
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 141.6,
+      "grad_norm": 0.5373359322547913,
+      "kl": 0.80438631772995,
+      "learning_rate": 3.4162530668107435e-06,
+      "loss": 0.0322,
+      "reward": 3.0027952194213867,
+      "reward_std": 0.3596634864807129,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.25279513001441956,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 708
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 141.8,
+      "grad_norm": 0.8107948303222656,
+      "kl": 0.7092552185058594,
+      "learning_rate": 3.4111762496978753e-06,
+      "loss": 0.0284,
+      "reward": 0.7482374310493469,
+      "reward_std": 2.8443443775177,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.2517626285552979,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 709
+    },
+    {
+      "completion_length": 484.0,
+      "epoch": 142.0,
+      "grad_norm": 0.5415573120117188,
+      "kl": 0.9022457599639893,
+      "learning_rate": 3.406095095709254e-06,
+      "loss": 0.0361,
+      "reward": 1.4290342330932617,
+      "reward_std": 1.759615182876587,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9772727489471436,
+      "rewards/wrapped_driving_reward": -1.5482385158538818,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 710
+    },
+    {
+      "completion_length": 474.0,
+      "epoch": 142.2,
+      "grad_norm": 0.4888540804386139,
+      "kl": 0.6180402636528015,
+      "learning_rate": 3.401009629029375e-06,
+      "loss": 0.0247,
+      "reward": 2.0353569984436035,
+      "reward_std": 2.0499258041381836,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9722222089767456,
+      "rewards/wrapped_driving_reward": -0.9368650317192078,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 711
+    },
+    {
+      "completion_length": 467.0,
+      "epoch": 142.4,
+      "grad_norm": 0.5631850957870483,
+      "kl": 1.0096300840377808,
+      "learning_rate": 3.39591987386325e-06,
+      "loss": 0.0404,
+      "reward": 1.2916650772094727,
+      "reward_std": 2.871446371078491,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.0833349227905273,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 712
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 142.6,
+      "grad_norm": 0.6677610278129578,
+      "kl": 0.7574449777603149,
+      "learning_rate": 3.3908258544363145e-06,
+      "loss": 0.0303,
+      "reward": -0.266355037689209,
+      "reward_std": 3.46443247795105,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.4583333134651184,
+      "rewards/wrapped_driving_reward": -1.9746882915496826,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 713
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 142.8,
+      "grad_norm": 0.4841836988925934,
+      "kl": 0.5822250247001648,
+      "learning_rate": 3.3857275949942896e-06,
+      "loss": 0.0233,
+      "reward": 3.0125911235809326,
+      "reward_std": 0.5904353260993958,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.13759109377861023,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 714
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 143.0,
+      "grad_norm": 0.6958256363868713,
+      "kl": 0.4327092468738556,
+      "learning_rate": 3.3806251198030843e-06,
+      "loss": 0.0173,
+      "reward": 1.4330600500106812,
+      "reward_std": 2.3058290481567383,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9545454978942871,
+      "rewards/wrapped_driving_reward": -1.0214855670928955,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 715
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 143.2,
+      "grad_norm": 0.6068946719169617,
+      "kl": 0.7822107076644897,
+      "learning_rate": 3.375518453148669e-06,
+      "loss": 0.0313,
+      "reward": 1.1330852508544922,
+      "reward_std": 2.4693946838378906,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.8669147491455078,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 716
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 143.4,
+      "grad_norm": 0.5714890956878662,
+      "kl": 0.5352396965026855,
+      "learning_rate": 3.370407619336966e-06,
+      "loss": 0.0214,
+      "reward": 2.959530830383301,
+      "reward_std": 0.21941396594047546,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.859375,
+      "rewards/wrapped_driving_reward": 0.2251558005809784,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 717
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 143.6,
+      "grad_norm": 0.7257641553878784,
+      "kl": 0.8822948336601257,
+      "learning_rate": 3.3652926426937327e-06,
+      "loss": 0.0353,
+      "reward": 3.610146999359131,
+      "reward_std": 0.20129412412643433,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.6101469397544861,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 718
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 143.8,
+      "grad_norm": 0.5029004812240601,
+      "kl": 1.3567814826965332,
+      "learning_rate": 3.360173547564442e-06,
+      "loss": 0.0543,
+      "reward": 1.8939110040664673,
+      "reward_std": 1.931662917137146,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.1060889959335327,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 719
+    },
+    {
+      "completion_length": 481.0,
+      "epoch": 144.0,
+      "grad_norm": 0.4105921685695648,
+      "kl": 1.52871572971344,
+      "learning_rate": 3.3550503583141726e-06,
+      "loss": 0.0611,
+      "reward": 3.451643943786621,
+      "reward_std": 0.4683004915714264,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.4516439437866211,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 720
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 144.2,
+      "grad_norm": 0.7768301367759705,
+      "kl": 1.18900465965271,
+      "learning_rate": 3.3499230993274857e-06,
+      "loss": 0.0476,
+      "reward": 0.9932074546813965,
+      "reward_std": 3.367805242538452,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.006792664527893,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 721
+    },
+    {
+      "completion_length": 490.0,
+      "epoch": 144.4,
+      "grad_norm": 0.6140910983085632,
+      "kl": 0.9986215829849243,
+      "learning_rate": 3.344791795008318e-06,
+      "loss": 0.0399,
+      "reward": 3.1455395221710205,
+      "reward_std": 0.6940726637840271,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.39553946256637573,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 722
+    },
+    {
+      "completion_length": 468.0,
+      "epoch": 144.6,
+      "grad_norm": 0.7327135801315308,
+      "kl": 1.4013735055923462,
+      "learning_rate": 3.339656469779856e-06,
+      "loss": 0.0561,
+      "reward": 2.5551810264587402,
+      "reward_std": 0.48002633452415466,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.19481904804706573,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 723
+    },
+    {
+      "completion_length": 428.0,
+      "epoch": 144.8,
+      "grad_norm": 0.5205874443054199,
+      "kl": 1.3122892379760742,
+      "learning_rate": 3.3345171480844275e-06,
+      "loss": 0.0525,
+      "reward": 3.3511462211608887,
+      "reward_std": 0.05403972789645195,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.35114631056785583,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 724
+    },
+    {
+      "completion_length": 430.0,
+      "epoch": 145.0,
+      "grad_norm": 0.5604906678199768,
+      "kl": 0.6418074369430542,
+      "learning_rate": 3.3293738543833807e-06,
+      "loss": 0.0257,
+      "reward": 3.1239919662475586,
+      "reward_std": 0.22990824282169342,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9090908765792847,
+      "rewards/wrapped_driving_reward": 0.33990123867988586,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 725
+    },
+    {
+      "completion_length": 362.0,
+      "epoch": 145.2,
+      "grad_norm": 0.6044005155563354,
+      "kl": 1.2028814554214478,
+      "learning_rate": 3.3242266131569685e-06,
+      "loss": 0.0481,
+      "reward": 3.8261451721191406,
+      "reward_std": 0.014617701061069965,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8261451721191406,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 726
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 145.4,
+      "grad_norm": 0.6275110244750977,
+      "kl": 0.6762914657592773,
+      "learning_rate": 3.3190754489042343e-06,
+      "loss": 0.0271,
+      "reward": 2.7210917472839355,
+      "reward_std": 0.9790669679641724,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9772727489471436,
+      "rewards/wrapped_driving_reward": 0.11881905049085617,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 727
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 145.6,
+      "grad_norm": 0.5946706533432007,
+      "kl": 0.5865952372550964,
+      "learning_rate": 3.313920386142892e-06,
+      "loss": 0.0235,
+      "reward": 2.5557713508605957,
+      "reward_std": 1.0863637924194336,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.3192285895347595,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 728
+    },
+    {
+      "completion_length": 492.0,
+      "epoch": 145.8,
+      "grad_norm": 0.5380379557609558,
+      "kl": 1.6467604637145996,
+      "learning_rate": 3.308761449409213e-06,
+      "loss": 0.0659,
+      "reward": 1.8841032981872559,
+      "reward_std": 1.9241235256195068,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.1158965826034546,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 729
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 146.0,
+      "grad_norm": 0.6382113099098206,
+      "kl": 0.7868921160697937,
+      "learning_rate": 3.303598663257904e-06,
+      "loss": 0.0315,
+      "reward": 1.3918712139129639,
+      "reward_std": 3.2618727684020996,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7083333134651184,
+      "rewards/wrapped_driving_reward": -0.9414620995521545,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 730
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 146.2,
+      "grad_norm": 0.5164018273353577,
+      "kl": 1.0673645734786987,
+      "learning_rate": 3.298432052261998e-06,
+      "loss": 0.0427,
+      "reward": 2.141380548477173,
+      "reward_std": 0.5501049160957336,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9146825075149536,
+      "rewards/wrapped_driving_reward": -0.39830195903778076,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 731
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 146.4,
+      "grad_norm": 0.493426114320755,
+      "kl": 0.8695380687713623,
+      "learning_rate": 3.293261641012731e-06,
+      "loss": 0.0348,
+      "reward": 3.68026065826416,
+      "reward_std": 0.3121219873428345,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.6802605986595154,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 732
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 146.6,
+      "grad_norm": 0.5762398838996887,
+      "kl": 1.1291520595550537,
+      "learning_rate": 3.288087454119425e-06,
+      "loss": 0.0452,
+      "reward": 2.828002691268921,
+      "reward_std": 0.4208530783653259,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9642857313156128,
+      "rewards/wrapped_driving_reward": 0.23871691524982452,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 733
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 146.8,
+      "grad_norm": 0.4853045642375946,
+      "kl": 0.49225303530693054,
+      "learning_rate": 3.282909516209374e-06,
+      "loss": 0.0197,
+      "reward": 2.2937583923339844,
+      "reward_std": 1.2878106832504272,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.5812417268753052,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 734
+    },
+    {
+      "completion_length": 361.0,
+      "epoch": 147.0,
+      "grad_norm": 0.5176259279251099,
+      "kl": 0.6392626762390137,
+      "learning_rate": 3.277727851927727e-06,
+      "loss": 0.0256,
+      "reward": 2.8032870292663574,
+      "reward_std": 0.37582576274871826,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.07171304523944855,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 735
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 147.2,
+      "grad_norm": 0.5171045660972595,
+      "kl": 0.9497402310371399,
+      "learning_rate": 3.272542485937369e-06,
+      "loss": 0.038,
+      "reward": 1.833735704421997,
+      "reward_std": 3.556070566177368,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.5412643551826477,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 736
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 147.4,
+      "grad_norm": 0.5060833096504211,
+      "kl": 1.1487382650375366,
+      "learning_rate": 3.2673534429188005e-06,
+      "loss": 0.0459,
+      "reward": 1.8757826089859009,
+      "reward_std": 0.8785591125488281,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9750000238418579,
+      "rewards/wrapped_driving_reward": -0.849217414855957,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 737
+    },
+    {
+      "completion_length": 401.0,
+      "epoch": 147.6,
+      "grad_norm": 0.5445796251296997,
+      "kl": 1.0297980308532715,
+      "learning_rate": 3.2621607475700272e-06,
+      "loss": 0.0412,
+      "reward": 3.600897789001465,
+      "reward_std": 0.257000595331192,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.7258977890014648,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 738
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 147.8,
+      "grad_norm": 0.640143096446991,
+      "kl": 0.9937734603881836,
+      "learning_rate": 3.256964424606437e-06,
+      "loss": 0.0398,
+      "reward": 1.05731201171875,
+      "reward_std": 3.0485732555389404,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.6875,
+      "rewards/wrapped_driving_reward": -1.25518798828125,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 739
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 148.0,
+      "grad_norm": 0.5605321526527405,
+      "kl": 0.5616591572761536,
+      "learning_rate": 3.2517644987606827e-06,
+      "loss": 0.0225,
+      "reward": 0.9253636598587036,
+      "reward_std": 2.987279176712036,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.074636459350586,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 740
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 148.2,
+      "grad_norm": 0.5340696573257446,
+      "kl": 0.4105786383152008,
+      "learning_rate": 3.2465609947825692e-06,
+      "loss": 0.0164,
+      "reward": 2.3854103088378906,
+      "reward_std": 0.5500273108482361,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9166666865348816,
+      "rewards/wrapped_driving_reward": -0.031256303191185,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 741
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 148.4,
+      "grad_norm": 0.5821884870529175,
+      "kl": 1.4779444932937622,
+      "learning_rate": 3.2413539374389275e-06,
+      "loss": 0.0591,
+      "reward": 0.9070387482643127,
+      "reward_std": 2.9707443714141846,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.6875,
+      "rewards/wrapped_driving_reward": -1.155461311340332,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 742
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 148.6,
+      "grad_norm": 0.5381807684898376,
+      "kl": 1.0403640270233154,
+      "learning_rate": 3.2361433515135053e-06,
+      "loss": 0.0416,
+      "reward": 1.9871805906295776,
+      "reward_std": 3.6582367420196533,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7361111044883728,
+      "rewards/wrapped_driving_reward": -0.373930424451828,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 743
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 148.8,
+      "grad_norm": 0.5100287795066833,
+      "kl": 1.2638732194900513,
+      "learning_rate": 3.230929261806842e-06,
+      "loss": 0.0506,
+      "reward": 3.228283405303955,
+      "reward_std": 0.640960693359375,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.4782832860946655,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 744
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 149.0,
+      "grad_norm": 0.501332700252533,
+      "kl": 1.4255468845367432,
+      "learning_rate": 3.225711693136156e-06,
+      "loss": 0.057,
+      "reward": 1.001197338104248,
+      "reward_std": 2.017551898956299,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.8738027811050415,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 745
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 149.2,
+      "grad_norm": 0.5337965488433838,
+      "kl": 1.2356843948364258,
+      "learning_rate": 3.2204906703352236e-06,
+      "loss": 0.0494,
+      "reward": 2.2676119804382324,
+      "reward_std": 0.3297145366668701,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.48238787055015564,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 746
+    },
+    {
+      "completion_length": 388.0,
+      "epoch": 149.4,
+      "grad_norm": 0.5865975022315979,
+      "kl": 1.1721208095550537,
+      "learning_rate": 3.215266218254261e-06,
+      "loss": 0.0469,
+      "reward": 3.2063422203063965,
+      "reward_std": 0.2641158699989319,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.20634236931800842,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 747
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 149.6,
+      "grad_norm": 0.569024920463562,
+      "kl": 0.868983805179596,
+      "learning_rate": 3.2100383617598075e-06,
+      "loss": 0.0348,
+      "reward": 2.9906487464904785,
+      "reward_std": 0.3553977310657501,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9772727489471436,
+      "rewards/wrapped_driving_reward": 0.763375997543335,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 748
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 149.8,
+      "grad_norm": 0.5583156943321228,
+      "kl": 1.0145978927612305,
+      "learning_rate": 3.2048071257346043e-06,
+      "loss": 0.0406,
+      "reward": 3.5098564624786377,
+      "reward_std": 0.30953365564346313,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.6348564624786377,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 749
+    },
+    {
+      "completion_length": 383.0,
+      "epoch": 150.0,
+      "grad_norm": 0.792302668094635,
+      "kl": 0.8888383507728577,
+      "learning_rate": 3.199572535077481e-06,
+      "loss": 0.0356,
+      "reward": 0.08345681428909302,
+      "reward_std": 2.1669135093688965,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -2.9165432453155518,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 750
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 150.2,
+      "grad_norm": 0.5216514468193054,
+      "kl": 0.9542403817176819,
+      "learning_rate": 3.194334614703231e-06,
+      "loss": 0.0382,
+      "reward": -0.3409658670425415,
+      "reward_std": 3.7293381690979004,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.8409658670425415,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 751
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 150.4,
+      "grad_norm": 0.4775712788105011,
+      "kl": 1.2860143184661865,
+      "learning_rate": 3.189093389542498e-06,
+      "loss": 0.0514,
+      "reward": 1.3992867469787598,
+      "reward_std": 1.6630216836929321,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.2257131338119507,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 752
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 150.6,
+      "grad_norm": 0.49834057688713074,
+      "kl": 0.9300306439399719,
+      "learning_rate": 3.183848884541656e-06,
+      "loss": 0.0372,
+      "reward": 2.7901477813720703,
+      "reward_std": 1.1417899131774902,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.29014769196510315,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 753
+    },
+    {
+      "completion_length": 404.0,
+      "epoch": 150.8,
+      "grad_norm": 0.5298438668251038,
+      "kl": 1.067299485206604,
+      "learning_rate": 3.1786011246626858e-06,
+      "loss": 0.0427,
+      "reward": 2.8012261390686035,
+      "reward_std": 0.03125178441405296,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.1987738460302353,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 754
+    },
+    {
+      "completion_length": 483.0,
+      "epoch": 151.0,
+      "grad_norm": 0.5239464640617371,
+      "kl": 1.0353827476501465,
+      "learning_rate": 3.173350134883066e-06,
+      "loss": 0.0414,
+      "reward": 2.897402763366699,
+      "reward_std": 0.7444739937782288,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8999999761581421,
+      "rewards/wrapped_driving_reward": -0.002597332000732422,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 755
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 151.2,
+      "grad_norm": 0.5170385241508484,
+      "kl": 0.5912831425666809,
+      "learning_rate": 3.1680959401956425e-06,
+      "loss": 0.0237,
+      "reward": 2.6039931774139404,
+      "reward_std": 0.9651803970336914,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.39600682258605957,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 756
+    },
+    {
+      "completion_length": 470.0,
+      "epoch": 151.4,
+      "grad_norm": 0.5630961656570435,
+      "kl": 1.3199890851974487,
+      "learning_rate": 3.1628385656085204e-06,
+      "loss": 0.0528,
+      "reward": 0.8596276640892029,
+      "reward_std": 1.6715515851974487,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -2.0153722763061523,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 757
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 151.6,
+      "grad_norm": 0.5590241551399231,
+      "kl": 0.7221583724021912,
+      "learning_rate": 3.157578036144937e-06,
+      "loss": 0.0289,
+      "reward": 2.7205729484558105,
+      "reward_std": 0.37680330872535706,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.1544271856546402,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 758
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 151.8,
+      "grad_norm": 0.6123512387275696,
+      "kl": 0.9747918844223022,
+      "learning_rate": 3.1523143768431475e-06,
+      "loss": 0.039,
+      "reward": 1.3794039487838745,
+      "reward_std": 3.6140682697296143,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7395833134651184,
+      "rewards/wrapped_driving_reward": -0.48517945408821106,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 759
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 152.0,
+      "grad_norm": 0.5005034804344177,
+      "kl": 0.639513373374939,
+      "learning_rate": 3.147047612756302e-06,
+      "loss": 0.0256,
+      "reward": 2.4191792011260986,
+      "reward_std": 1.3698595762252808,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.20582084357738495,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 760
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 152.2,
+      "grad_norm": 0.4641207158565521,
+      "kl": 1.0183018445968628,
+      "learning_rate": 3.1417777689523297e-06,
+      "loss": 0.0407,
+      "reward": 1.737586259841919,
+      "reward_std": 3.4952027797698975,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.6374138593673706,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 761
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 152.4,
+      "grad_norm": 0.5382841229438782,
+      "kl": 0.6911734342575073,
+      "learning_rate": 3.136504870513819e-06,
+      "loss": 0.0276,
+      "reward": 3.6939151287078857,
+      "reward_std": 0.23970358073711395,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8189151287078857,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 762
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 152.6,
+      "grad_norm": 0.5685204863548279,
+      "kl": 1.0369704961776733,
+      "learning_rate": 3.131228942537895e-06,
+      "loss": 0.0415,
+      "reward": 3.4881484508514404,
+      "reward_std": 0.1292411834001541,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9583333134651184,
+      "rewards/wrapped_driving_reward": 0.5298152565956116,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 763
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 152.8,
+      "grad_norm": 0.4972493648529053,
+      "kl": 1.0108205080032349,
+      "learning_rate": 3.125950010136104e-06,
+      "loss": 0.0404,
+      "reward": 2.9704365730285645,
+      "reward_std": 0.15474218130111694,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9166666865348816,
+      "rewards/wrapped_driving_reward": 0.05377001687884331,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 764
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 153.0,
+      "grad_norm": 0.5436493158340454,
+      "kl": 0.8178736567497253,
+      "learning_rate": 3.120668098434291e-06,
+      "loss": 0.0327,
+      "reward": -1.375,
+      "reward_std": 0.4787135720252991,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 765
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 153.2,
+      "grad_norm": 1.6990164518356323,
+      "kl": 1.2837238311767578,
+      "learning_rate": 3.115383232572483e-06,
+      "loss": 0.0513,
+      "reward": 0.06262272596359253,
+      "reward_std": 1.4747098684310913,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -2.8123772144317627,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 766
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 153.4,
+      "grad_norm": 0.6486381888389587,
+      "kl": 0.36142438650131226,
+      "learning_rate": 3.1100954377047665e-06,
+      "loss": 0.0145,
+      "reward": 1.3286774158477783,
+      "reward_std": 3.5647871494293213,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9213225245475769,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 767
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 153.6,
+      "grad_norm": 0.6793121695518494,
+      "kl": 1.1117873191833496,
+      "learning_rate": 3.1048047389991693e-06,
+      "loss": 0.0445,
+      "reward": 3.3907761573791504,
+      "reward_std": 0.20889942348003387,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.5157762765884399,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 768
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 153.8,
+      "grad_norm": 0.4357717037200928,
+      "kl": 0.67461097240448,
+      "learning_rate": 3.0995111616375417e-06,
+      "loss": 0.027,
+      "reward": 1.8588078022003174,
+      "reward_std": 3.5830955505371094,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.5161921381950378,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 769
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 154.0,
+      "grad_norm": 0.48567458987236023,
+      "kl": 0.8509070873260498,
+      "learning_rate": 3.094214730815433e-06,
+      "loss": 0.034,
+      "reward": 2.213784694671631,
+      "reward_std": 0.4306184649467468,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9375,
+      "rewards/wrapped_driving_reward": -0.22371524572372437,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 770
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 154.2,
+      "grad_norm": 0.826822817325592,
+      "kl": 0.3968334496021271,
+      "learning_rate": 3.088915471741976e-06,
+      "loss": 0.0159,
+      "reward": 2.260512351989746,
+      "reward_std": 0.8328320384025574,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8500000238418579,
+      "rewards/wrapped_driving_reward": -0.21448758244514465,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 771
+    },
+    {
+      "completion_length": 461.0,
+      "epoch": 154.4,
+      "grad_norm": 0.5886190533638,
+      "kl": 1.100509524345398,
+      "learning_rate": 3.0836134096397642e-06,
+      "loss": 0.044,
+      "reward": 3.2229135036468506,
+      "reward_std": 0.6925670504570007,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.3479134142398834,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 772
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 154.6,
+      "grad_norm": 0.5568466782569885,
+      "kl": 0.4733794629573822,
+      "learning_rate": 3.0783085697447324e-06,
+      "loss": 0.0189,
+      "reward": 1.964868187904358,
+      "reward_std": 1.187056303024292,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.9101318120956421,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 773
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 154.8,
+      "grad_norm": 1.7119587659835815,
+      "kl": 1.4648423194885254,
+      "learning_rate": 3.073000977306036e-06,
+      "loss": 0.0586,
+      "reward": 2.845578908920288,
+      "reward_std": 0.34594619274139404,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.02942105382680893,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 774
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 155.0,
+      "grad_norm": 0.5909410715103149,
+      "kl": 0.5023986101150513,
+      "learning_rate": 3.0676906575859335e-06,
+      "loss": 0.0201,
+      "reward": 2.984121799468994,
+      "reward_std": 0.6469627022743225,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9027777314186096,
+      "rewards/wrapped_driving_reward": 0.45634397864341736,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 775
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 155.2,
+      "grad_norm": 0.46868669986724854,
+      "kl": 1.4069966077804565,
+      "learning_rate": 3.062377635859663e-06,
+      "loss": 0.0563,
+      "reward": 2.647207736968994,
+      "reward_std": 0.2306966334581375,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9722222089767456,
+      "rewards/wrapped_driving_reward": -0.20001450181007385,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 776
+    },
+    {
+      "completion_length": 471.0,
+      "epoch": 155.4,
+      "grad_norm": 0.4950220286846161,
+      "kl": 1.027194857597351,
+      "learning_rate": 3.0570619374153234e-06,
+      "loss": 0.0411,
+      "reward": 3.526076316833496,
+      "reward_std": 0.2638986110687256,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9642857313156128,
+      "rewards/wrapped_driving_reward": 0.8117905259132385,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 777
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 155.6,
+      "grad_norm": 0.4979739487171173,
+      "kl": 0.7722088694572449,
+      "learning_rate": 3.051743587553754e-06,
+      "loss": 0.0309,
+      "reward": 3.0607147216796875,
+      "reward_std": 0.5555264949798584,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.18571479618549347,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 778
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 155.8,
+      "grad_norm": 0.5324958562850952,
+      "kl": 0.9706301689147949,
+      "learning_rate": 3.0464226115884115e-06,
+      "loss": 0.0388,
+      "reward": 1.3502414226531982,
+      "reward_std": 2.969165563583374,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.7747586369514465,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 779
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 156.0,
+      "grad_norm": 0.5909656286239624,
+      "kl": 0.8194612860679626,
+      "learning_rate": 3.0410990348452572e-06,
+      "loss": 0.0328,
+      "reward": 1.87015700340271,
+      "reward_std": 1.9332139492034912,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.1298428773880005,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 780
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 156.2,
+      "grad_norm": 0.664778470993042,
+      "kl": 0.9126893281936646,
+      "learning_rate": 3.035772882662627e-06,
+      "loss": 0.0365,
+      "reward": 3.700016975402832,
+      "reward_std": 0.2690635919570923,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8250167965888977,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 781
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 156.4,
+      "grad_norm": 0.4898008704185486,
+      "kl": 0.34648996591567993,
+      "learning_rate": 3.030444180391116e-06,
+      "loss": 0.0139,
+      "reward": 1.926034688949585,
+      "reward_std": 0.2995733916759491,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9772727489471436,
+      "rewards/wrapped_driving_reward": -0.1762380450963974,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 782
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 156.6,
+      "grad_norm": 0.7782924175262451,
+      "kl": 1.022233247756958,
+      "learning_rate": 3.0251129533934565e-06,
+      "loss": 0.0409,
+      "reward": 3.3980164527893066,
+      "reward_std": 0.24236531555652618,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9642857313156128,
+      "rewards/wrapped_driving_reward": 0.4337307810783386,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 783
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 156.8,
+      "grad_norm": 0.6505765914916992,
+      "kl": 0.8364863991737366,
+      "learning_rate": 3.019779227044398e-06,
+      "loss": 0.0335,
+      "reward": 3.134411573410034,
+      "reward_std": 0.42541173100471497,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.96875,
+      "rewards/wrapped_driving_reward": 0.29066160321235657,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 784
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 157.0,
+      "grad_norm": 0.508212685585022,
+      "kl": 0.38002079725265503,
+      "learning_rate": 3.0144430267305874e-06,
+      "loss": 0.0152,
+      "reward": 3.031036376953125,
+      "reward_std": 0.23074941337108612,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.0310362558811903,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 785
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 157.2,
+      "grad_norm": 0.5105311274528503,
+      "kl": 0.2681836783885956,
+      "learning_rate": 3.0091043778504438e-06,
+      "loss": 0.0107,
+      "reward": -2.249131917953491,
+      "reward_std": 3.5017361640930176,
+      "rewards/mpc_param_extraction_reward": 0.25,
+      "rewards/mpc_param_name_reward": 0.25,
+      "rewards/wrapped_driving_reward": -2.999131917953491,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 786
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 157.4,
+      "grad_norm": 0.5429096221923828,
+      "kl": 0.4873754680156708,
+      "learning_rate": 3.0037633058140433e-06,
+      "loss": 0.0195,
+      "reward": 3.7257652282714844,
+      "reward_std": 0.21732982993125916,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.7257651090621948,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 787
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 157.6,
+      "grad_norm": 0.6699551343917847,
+      "kl": 0.7164745926856995,
+      "learning_rate": 2.998419836042993e-06,
+      "loss": 0.0287,
+      "reward": 3.4064254760742188,
+      "reward_std": 0.2806980609893799,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.5314255952835083,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 788
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 157.8,
+      "grad_norm": 0.4117250442504883,
+      "kl": 1.1326197385787964,
+      "learning_rate": 2.993073993970316e-06,
+      "loss": 0.0453,
+      "reward": 2.703610897064209,
+      "reward_std": 0.4096671938896179,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.29638922214508057,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 789
+    },
+    {
+      "completion_length": 355.0,
+      "epoch": 158.0,
+      "grad_norm": 0.5928639769554138,
+      "kl": 0.5766419768333435,
+      "learning_rate": 2.9877258050403214e-06,
+      "loss": 0.0231,
+      "reward": 3.0424065589904785,
+      "reward_std": 0.2919481694698334,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.04240674898028374,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 790
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 158.2,
+      "grad_norm": 0.5171490907669067,
+      "kl": 0.8181880712509155,
+      "learning_rate": 2.9823752947084926e-06,
+      "loss": 0.0327,
+      "reward": 3.0578365325927734,
+      "reward_std": 0.42794013023376465,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.307836651802063,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 791
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 158.4,
+      "grad_norm": 0.5789440870285034,
+      "kl": 0.6994017958641052,
+      "learning_rate": 2.9770224884413625e-06,
+      "loss": 0.028,
+      "reward": 2.8126344680786133,
+      "reward_std": 0.8600202202796936,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.18763434886932373,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 792
+    },
+    {
+      "completion_length": 383.0,
+      "epoch": 158.6,
+      "grad_norm": 0.7086771130561829,
+      "kl": 1.1586806774139404,
+      "learning_rate": 2.9716674117163886e-06,
+      "loss": 0.0463,
+      "reward": 1.134717583656311,
+      "reward_std": 2.779279947280884,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.240282416343689,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 793
+    },
+    {
+      "completion_length": 499.0,
+      "epoch": 158.8,
+      "grad_norm": 0.4350188970565796,
+      "kl": 0.5939379334449768,
+      "learning_rate": 2.966310090021837e-06,
+      "loss": 0.0237,
+      "reward": 3.8331589698791504,
+      "reward_std": 0.011624080128967762,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8331590890884399,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 794
+    },
+    {
+      "completion_length": 420.0,
+      "epoch": 159.0,
+      "grad_norm": 0.6273412704467773,
+      "kl": 0.8126957416534424,
+      "learning_rate": 2.9609505488566585e-06,
+      "loss": 0.0325,
+      "reward": 1.8683500289916992,
+      "reward_std": 2.0532751083374023,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.7833333611488342,
+      "rewards/wrapped_driving_reward": -0.7899832129478455,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 795
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 159.2,
+      "grad_norm": 0.7259314060211182,
+      "kl": 0.9824731945991516,
+      "learning_rate": 2.9555888137303695e-06,
+      "loss": 0.0393,
+      "reward": -1.75,
+      "reward_std": 1.1902379989624023,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 796
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 159.4,
+      "grad_norm": 0.9288682341575623,
+      "kl": 1.0356067419052124,
+      "learning_rate": 2.9502249101629248e-06,
+      "loss": 0.0414,
+      "reward": 3.432770013809204,
+      "reward_std": 0.463666170835495,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9772727489471436,
+      "rewards/wrapped_driving_reward": 0.8304973840713501,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 797
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 159.6,
+      "grad_norm": 0.6679523587226868,
+      "kl": 0.6253860592842102,
+      "learning_rate": 2.944858863684605e-06,
+      "loss": 0.025,
+      "reward": 2.203218936920166,
+      "reward_std": 2.1417882442474365,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.953125,
+      "rewards/wrapped_driving_reward": -0.4999062418937683,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 798
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 159.8,
+      "grad_norm": 0.7524790167808533,
+      "kl": 0.8385742902755737,
+      "learning_rate": 2.939490699835887e-06,
+      "loss": 0.0335,
+      "reward": 1.3603956699371338,
+      "reward_std": 3.2636396884918213,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.8896043300628662,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 799
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 160.0,
+      "grad_norm": 0.6979427337646484,
+      "kl": 0.27322471141815186,
+      "learning_rate": 2.9341204441673267e-06,
+      "loss": 0.0109,
+      "reward": -0.4729641079902649,
+      "reward_std": 3.7896523475646973,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.9729641675949097,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 800
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 160.2,
+      "grad_norm": 0.5591041445732117,
+      "kl": 1.02774178981781,
+      "learning_rate": 2.9287481222394358e-06,
+      "loss": 0.0411,
+      "reward": 3.2085986137390137,
+      "reward_std": 0.36301255226135254,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.5835985541343689,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 801
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 160.4,
+      "grad_norm": 0.5321090817451477,
+      "kl": 0.6709323525428772,
+      "learning_rate": 2.9233737596225616e-06,
+      "loss": 0.0268,
+      "reward": 0.336200475692749,
+      "reward_std": 3.2529170513153076,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.6722221970558167,
+      "rewards/wrapped_driving_reward": -1.5860216617584229,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 802
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 160.6,
+      "grad_norm": 0.7834795713424683,
+      "kl": 1.0378621816635132,
+      "learning_rate": 2.9179973818967643e-06,
+      "loss": 0.0415,
+      "reward": 1.797407627105713,
+      "reward_std": 3.538046360015869,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.45259231328964233,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 803
+    },
+    {
+      "completion_length": 400.0,
+      "epoch": 160.8,
+      "grad_norm": 0.530414342880249,
+      "kl": 1.1836286783218384,
+      "learning_rate": 2.912619014651694e-06,
+      "loss": 0.0473,
+      "reward": 2.7897591590881348,
+      "reward_std": 0.024412136524915695,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.21024088561534882,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 804
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 161.0,
+      "grad_norm": 0.7451685667037964,
+      "kl": 1.1952420473098755,
+      "learning_rate": 2.9072386834864723e-06,
+      "loss": 0.0478,
+      "reward": 2.9604384899139404,
+      "reward_std": 0.519312858581543,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9583333134651184,
+      "rewards/wrapped_driving_reward": 0.12710511684417725,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 805
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 161.2,
+      "grad_norm": 0.5481700301170349,
+      "kl": 1.0363727807998657,
+      "learning_rate": 2.9018564140095657e-06,
+      "loss": 0.0415,
+      "reward": 1.8664171695709229,
+      "reward_std": 3.5856990814208984,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.625,
+      "rewards/wrapped_driving_reward": -0.3835829496383667,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 806
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 161.4,
+      "grad_norm": 0.4871310889720917,
+      "kl": 0.572235107421875,
+      "learning_rate": 2.896472231838668e-06,
+      "loss": 0.0229,
+      "reward": 2.6908867359161377,
+      "reward_std": 0.48215678334236145,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.19088676571846008,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 807
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 161.6,
+      "grad_norm": 0.5225286483764648,
+      "kl": 1.6882096529006958,
+      "learning_rate": 2.8910861626005774e-06,
+      "loss": 0.0675,
+      "reward": 2.6616103649139404,
+      "reward_std": 0.28830450773239136,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9750000238418579,
+      "rewards/wrapped_driving_reward": -0.1883895993232727,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 808
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 161.8,
+      "grad_norm": 0.5506367683410645,
+      "kl": 0.40021976828575134,
+      "learning_rate": 2.8856982319310724e-06,
+      "loss": 0.016,
+      "reward": 0.8944512009620667,
+      "reward_std": 2.975062131881714,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.1055488586425781,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 809
+    },
+    {
+      "completion_length": 431.0,
+      "epoch": 162.0,
+      "grad_norm": 0.6307826042175293,
+      "kl": 0.8738049268722534,
+      "learning_rate": 2.880308465474792e-06,
+      "loss": 0.035,
+      "reward": 3.2625396251678467,
+      "reward_std": 0.23850522935390472,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.3875396251678467,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 810
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 162.2,
+      "grad_norm": 0.461091548204422,
+      "kl": 1.3140265941619873,
+      "learning_rate": 2.8749168888851126e-06,
+      "loss": 0.0526,
+      "reward": 2.7970871925354004,
+      "reward_std": 0.45916786789894104,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9583333134651184,
+      "rewards/wrapped_driving_reward": 0.21375392377376556,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 811
+    },
+    {
+      "completion_length": 399.0,
+      "epoch": 162.4,
+      "grad_norm": 0.5810427069664001,
+      "kl": 1.0196524858474731,
+      "learning_rate": 2.8695235278240272e-06,
+      "loss": 0.0408,
+      "reward": 3.457486629486084,
+      "reward_std": 0.3558793365955353,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.45748674869537354,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 812
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 162.6,
+      "grad_norm": 0.46939176321029663,
+      "kl": 0.9351792931556702,
+      "learning_rate": 2.8641284079620203e-06,
+      "loss": 0.0374,
+      "reward": 0.05920994281768799,
+      "reward_std": 2.118419885635376,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -2.9407901763916016,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 813
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 162.8,
+      "grad_norm": 0.528785228729248,
+      "kl": 0.5687186121940613,
+      "learning_rate": 2.858731554977948e-06,
+      "loss": 0.0227,
+      "reward": -0.43504947423934937,
+      "reward_std": 3.5419013500213623,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -2.060049295425415,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 814
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 163.0,
+      "grad_norm": 0.48642876744270325,
+      "kl": 0.55952388048172,
+      "learning_rate": 2.8533329945589192e-06,
+      "loss": 0.0224,
+      "reward": 0.7796438336372375,
+      "reward_std": 3.644176483154297,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.5953562259674072,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 815
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 163.2,
+      "grad_norm": 0.5921107530593872,
+      "kl": 0.8645963072776794,
+      "learning_rate": 2.847932752400164e-06,
+      "loss": 0.0346,
+      "reward": 3.478057861328125,
+      "reward_std": 0.34204110503196716,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.6030576825141907,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 816
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 163.4,
+      "grad_norm": 0.5188066959381104,
+      "kl": 0.912796676158905,
+      "learning_rate": 2.8425308542049208e-06,
+      "loss": 0.0365,
+      "reward": 2.615131139755249,
+      "reward_std": 0.12927274405956268,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9722222089767456,
+      "rewards/wrapped_driving_reward": -0.3570910692214966,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 817
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 163.6,
+      "grad_norm": 0.4986792206764221,
+      "kl": 0.6091123223304749,
+      "learning_rate": 2.837127325684308e-06,
+      "loss": 0.0244,
+      "reward": 0.9812687039375305,
+      "reward_std": 3.3806614875793457,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.2687312364578247,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 818
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 163.8,
+      "grad_norm": 0.6001780033111572,
+      "kl": 0.6903250217437744,
+      "learning_rate": 2.8317221925572058e-06,
+      "loss": 0.0276,
+      "reward": 2.1244254112243652,
+      "reward_std": 0.6480088233947754,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.2505744695663452,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 819
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 164.0,
+      "grad_norm": 0.4543159604072571,
+      "kl": 0.9673792123794556,
+      "learning_rate": 2.82631548055013e-06,
+      "loss": 0.0387,
+      "reward": 3.105719804763794,
+      "reward_std": 0.18213030695915222,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8333333134651184,
+      "rewards/wrapped_driving_reward": 0.5223863124847412,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 820
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 164.2,
+      "grad_norm": 0.7704506516456604,
+      "kl": 1.6314855813980103,
+      "learning_rate": 2.820907215397111e-06,
+      "loss": 0.0653,
+      "reward": 2.759028673171997,
+      "reward_std": 0.06708598881959915,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9772727489471436,
+      "rewards/wrapped_driving_reward": -0.21824395656585693,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 821
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 164.4,
+      "grad_norm": 0.5132427215576172,
+      "kl": 0.21008968353271484,
+      "learning_rate": 2.815497422839575e-06,
+      "loss": 0.0084,
+      "reward": 2.4701147079467773,
+      "reward_std": 0.5563299655914307,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9750000238418579,
+      "rewards/wrapped_driving_reward": -0.37988513708114624,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 822
+    },
+    {
+      "completion_length": 492.0,
+      "epoch": 164.6,
+      "grad_norm": 0.5533198118209839,
+      "kl": 1.597398281097412,
+      "learning_rate": 2.8100861286262137e-06,
+      "loss": 0.0639,
+      "reward": 3.173506021499634,
+      "reward_std": 0.10093068331480026,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9821428656578064,
+      "rewards/wrapped_driving_reward": 0.3163631558418274,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 823
+    },
+    {
+      "completion_length": 487.0,
+      "epoch": 164.8,
+      "grad_norm": 0.5174404978752136,
+      "kl": 1.3452297449111938,
+      "learning_rate": 2.804673358512869e-06,
+      "loss": 0.0538,
+      "reward": 3.791170120239258,
+      "reward_std": 0.12576456367969513,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.7911700010299683,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 824
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 165.0,
+      "grad_norm": 0.5471038818359375,
+      "kl": 1.292214274406433,
+      "learning_rate": 2.7992591382624064e-06,
+      "loss": 0.0517,
+      "reward": 3.421455144882202,
+      "reward_std": 0.5310376286506653,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.6714551448822021,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 825
+    },
+    {
+      "completion_length": 408.0,
+      "epoch": 165.2,
+      "grad_norm": 0.6716032028198242,
+      "kl": 0.8129616975784302,
+      "learning_rate": 2.7938434936445946e-06,
+      "loss": 0.0325,
+      "reward": 3.068939685821533,
+      "reward_std": 0.7359381914138794,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.5689395070075989,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 826
+    },
+    {
+      "completion_length": 368.0,
+      "epoch": 165.4,
+      "grad_norm": 0.7269822359085083,
+      "kl": 0.660163402557373,
+      "learning_rate": 2.78842645043598e-06,
+      "loss": 0.0264,
+      "reward": 3.6374306678771973,
+      "reward_std": 0.3847697377204895,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.6374306678771973,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 827
+    },
+    {
+      "completion_length": 433.0,
+      "epoch": 165.6,
+      "grad_norm": 0.49772003293037415,
+      "kl": 0.4905422031879425,
+      "learning_rate": 2.7830080344197675e-06,
+      "loss": 0.0196,
+      "reward": 2.9236674308776855,
+      "reward_std": 0.4421462118625641,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9772727489471436,
+      "rewards/wrapped_driving_reward": 0.19639474153518677,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 828
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 165.8,
+      "grad_norm": 0.5490583777427673,
+      "kl": 0.39918583631515503,
+      "learning_rate": 2.7775882713856946e-06,
+      "loss": 0.016,
+      "reward": 2.375999927520752,
+      "reward_std": 0.4072107970714569,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9772727489471436,
+      "rewards/wrapped_driving_reward": -0.22627297043800354,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 829
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 166.0,
+      "grad_norm": 0.5898881554603577,
+      "kl": 1.5381332635879517,
+      "learning_rate": 2.7721671871299115e-06,
+      "loss": 0.0615,
+      "reward": 0.9043681621551514,
+      "reward_std": 3.2865047454833984,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.6875,
+      "rewards/wrapped_driving_reward": -1.158131718635559,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 830
+    },
+    {
+      "completion_length": 441.0,
+      "epoch": 166.2,
+      "grad_norm": 0.5331966280937195,
+      "kl": 0.8137264251708984,
+      "learning_rate": 2.766744807454857e-06,
+      "loss": 0.0325,
+      "reward": 3.826152801513672,
+      "reward_std": 0.014650849625468254,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8261529803276062,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 831
+    },
+    {
+      "completion_length": 397.0,
+      "epoch": 166.4,
+      "grad_norm": 0.4468265771865845,
+      "kl": 0.6693469882011414,
+      "learning_rate": 2.761321158169134e-06,
+      "loss": 0.0268,
+      "reward": 1.760857343673706,
+      "reward_std": 1.9348771572113037,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.239142656326294,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 832
+    },
+    {
+      "completion_length": 365.0,
+      "epoch": 166.6,
+      "grad_norm": 0.6002892255783081,
+      "kl": 0.7958039045333862,
+      "learning_rate": 2.75589626508739e-06,
+      "loss": 0.0318,
+      "reward": 3.139150619506836,
+      "reward_std": 0.13503800332546234,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9166666865348816,
+      "rewards/wrapped_driving_reward": 0.2224840521812439,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 833
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 166.8,
+      "grad_norm": 0.6386081576347351,
+      "kl": 0.9411477446556091,
+      "learning_rate": 2.750470154030191e-06,
+      "loss": 0.0376,
+      "reward": 3.0650391578674316,
+      "reward_std": 0.6393033862113953,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.5650390386581421,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 834
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 167.0,
+      "grad_norm": 0.3837875723838806,
+      "kl": 1.657114028930664,
+      "learning_rate": 2.7450428508239024e-06,
+      "loss": 0.0663,
+      "reward": 2.6719822883605957,
+      "reward_std": 0.24073880910873413,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.20301775634288788,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 835
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 167.2,
+      "grad_norm": 0.506295919418335,
+      "kl": 0.7120727896690369,
+      "learning_rate": 2.7396143813005603e-06,
+      "loss": 0.0285,
+      "reward": 1.3677772283554077,
+      "reward_std": 3.2470555305480957,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7250000238418579,
+      "rewards/wrapped_driving_reward": -0.7322226762771606,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 836
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 167.4,
+      "grad_norm": 0.4985957741737366,
+      "kl": 0.9464759230613708,
+      "learning_rate": 2.734184771297756e-06,
+      "loss": 0.0379,
+      "reward": 2.5576024055480957,
+      "reward_std": 0.2574041485786438,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.06739747524261475,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 837
+    },
+    {
+      "completion_length": 439.0,
+      "epoch": 167.6,
+      "grad_norm": 0.5637804865837097,
+      "kl": 0.8566745519638062,
+      "learning_rate": 2.7287540466585067e-06,
+      "loss": 0.0343,
+      "reward": 2.9292807579040527,
+      "reward_std": 0.7874415516853333,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.0707191526889801,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 838
+    },
+    {
+      "completion_length": 409.0,
+      "epoch": 167.8,
+      "grad_norm": 0.5939984321594238,
+      "kl": 0.6225539445877075,
+      "learning_rate": 2.7233222332311344e-06,
+      "loss": 0.0249,
+      "reward": 3.39684796333313,
+      "reward_std": 0.42441999912261963,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.949999988079071,
+      "rewards/wrapped_driving_reward": 0.8218478560447693,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 839
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 168.0,
+      "grad_norm": 0.6109358668327332,
+      "kl": 0.8564927577972412,
+      "learning_rate": 2.717889356869146e-06,
+      "loss": 0.0343,
+      "reward": 3.2804532051086426,
+      "reward_std": 0.4187309145927429,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9722222089767456,
+      "rewards/wrapped_driving_reward": 0.3082309365272522,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 840
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 168.2,
+      "grad_norm": 0.5494788289070129,
+      "kl": 1.4602153301239014,
+      "learning_rate": 2.7124554434311047e-06,
+      "loss": 0.0584,
+      "reward": 1.7961541414260864,
+      "reward_std": 2.5778276920318604,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9583333134651184,
+      "rewards/wrapped_driving_reward": -0.662179172039032,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 841
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 168.4,
+      "grad_norm": 0.4992530345916748,
+      "kl": 1.0213444232940674,
+      "learning_rate": 2.707020518780511e-06,
+      "loss": 0.0409,
+      "reward": 2.7865848541259766,
+      "reward_std": 0.029527578502893448,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.21341529488563538,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 842
+    },
+    {
+      "completion_length": 458.0,
+      "epoch": 168.6,
+      "grad_norm": 0.5227845311164856,
+      "kl": 0.5304321646690369,
+      "learning_rate": 2.7015846087856796e-06,
+      "loss": 0.0212,
+      "reward": 1.1565890312194824,
+      "reward_std": 2.35237979888916,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9375,
+      "rewards/wrapped_driving_reward": -1.7809109687805176,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 843
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 168.8,
+      "grad_norm": 0.5163724422454834,
+      "kl": 1.1655185222625732,
+      "learning_rate": 2.696147739319613e-06,
+      "loss": 0.0466,
+      "reward": 1.2925224304199219,
+      "reward_std": 3.271446704864502,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9574775099754333,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 844
+    },
+    {
+      "completion_length": 391.0,
+      "epoch": 169.0,
+      "grad_norm": 0.5346372127532959,
+      "kl": 0.9912291765213013,
+      "learning_rate": 2.6907099362598815e-06,
+      "loss": 0.0396,
+      "reward": 3.68274188041687,
+      "reward_std": 0.24550725519657135,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9722222089767456,
+      "rewards/wrapped_driving_reward": 0.835519552230835,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 845
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 169.2,
+      "grad_norm": 0.4762868285179138,
+      "kl": 0.9365013837814331,
+      "learning_rate": 2.6852712254884988e-06,
+      "loss": 0.0375,
+      "reward": 3.4418392181396484,
+      "reward_std": 0.48263806104660034,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9821428656578064,
+      "rewards/wrapped_driving_reward": 0.8346964716911316,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 846
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 169.4,
+      "grad_norm": 0.5167540311813354,
+      "kl": 0.4593685567378998,
+      "learning_rate": 2.6798316328917988e-06,
+      "loss": 0.0184,
+      "reward": 2.625422477722168,
+      "reward_std": 0.3436520993709564,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.12457744032144547,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 847
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 169.6,
+      "grad_norm": 0.5815231204032898,
+      "kl": 1.4257601499557495,
+      "learning_rate": 2.6743911843603134e-06,
+      "loss": 0.057,
+      "reward": 2.8921163082122803,
+      "reward_std": 0.4228496849536896,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9750000238418579,
+      "rewards/wrapped_driving_reward": 0.16711634397506714,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 848
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 169.8,
+      "grad_norm": 0.6285993456840515,
+      "kl": 1.027273178100586,
+      "learning_rate": 2.6689499057886483e-06,
+      "loss": 0.0411,
+      "reward": 1.422483205795288,
+      "reward_std": 3.62673020362854,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.7025167942047119,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 849
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 170.0,
+      "grad_norm": 0.5311692953109741,
+      "kl": 0.724616527557373,
+      "learning_rate": 2.663507823075358e-06,
+      "loss": 0.029,
+      "reward": 0.08719003200531006,
+      "reward_std": 2.17438006401062,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -2.9128098487854004,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 850
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 170.2,
+      "grad_norm": 0.6036850810050964,
+      "kl": 0.88832688331604,
+      "learning_rate": 2.6580649621228267e-06,
+      "loss": 0.0355,
+      "reward": 0.6683868169784546,
+      "reward_std": 3.1441006660461426,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.6666666865348816,
+      "rewards/wrapped_driving_reward": -1.2482798099517822,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 851
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 170.4,
+      "grad_norm": 0.5897907614707947,
+      "kl": 1.2000190019607544,
+      "learning_rate": 2.6526213488371427e-06,
+      "loss": 0.048,
+      "reward": 2.9573283195495605,
+      "reward_std": 0.03875737264752388,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.08232828229665756,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 852
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 170.6,
+      "grad_norm": 0.5143535137176514,
+      "kl": 0.4949548840522766,
+      "learning_rate": 2.6471770091279725e-06,
+      "loss": 0.0198,
+      "reward": 2.131885528564453,
+      "reward_std": 0.730937123298645,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.4931145906448364,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 853
+    },
+    {
+      "completion_length": 393.0,
+      "epoch": 170.8,
+      "grad_norm": 0.5587767958641052,
+      "kl": 1.4226547479629517,
+      "learning_rate": 2.641731968908444e-06,
+      "loss": 0.0569,
+      "reward": 2.96309232711792,
+      "reward_std": 0.2758857309818268,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9375,
+      "rewards/wrapped_driving_reward": 0.15059247612953186,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 854
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 171.0,
+      "grad_norm": 0.49916645884513855,
+      "kl": 0.8778588771820068,
+      "learning_rate": 2.6362862540950163e-06,
+      "loss": 0.0351,
+      "reward": 3.481083631515503,
+      "reward_std": 0.4503914415836334,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.7310836315155029,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 855
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 171.2,
+      "grad_norm": 0.5146768689155579,
+      "kl": 0.9753352403640747,
+      "learning_rate": 2.6308398906073603e-06,
+      "loss": 0.039,
+      "reward": 3.0682339668273926,
+      "reward_std": 0.6612880825996399,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.31823405623435974,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 856
+    },
+    {
+      "completion_length": 417.0,
+      "epoch": 171.4,
+      "grad_norm": 1.3383334875106812,
+      "kl": 0.49587488174438477,
+      "learning_rate": 2.6253929043682336e-06,
+      "loss": 0.0198,
+      "reward": 3.442657470703125,
+      "reward_std": 0.5057659149169922,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.817657470703125,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 857
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 171.6,
+      "grad_norm": 1.6063464879989624,
+      "kl": 1.655110239982605,
+      "learning_rate": 2.61994532130336e-06,
+      "loss": 0.0662,
+      "reward": 2.950542449951172,
+      "reward_std": 0.4103209972381592,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.96875,
+      "rewards/wrapped_driving_reward": 0.10679234564304352,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 858
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 171.8,
+      "grad_norm": 0.5413594245910645,
+      "kl": 0.9937121272087097,
+      "learning_rate": 2.6144971673413023e-06,
+      "loss": 0.0397,
+      "reward": 2.7468223571777344,
+      "reward_std": 0.6185001134872437,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.003177560865879059,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 859
+    },
+    {
+      "completion_length": 405.0,
+      "epoch": 172.0,
+      "grad_norm": 0.5254268646240234,
+      "kl": 0.7705658674240112,
+      "learning_rate": 2.6090484684133406e-06,
+      "loss": 0.0308,
+      "reward": 1.0588258504867554,
+      "reward_std": 2.706279993057251,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7272727489471436,
+      "rewards/wrapped_driving_reward": -1.4184470176696777,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 860
+    },
+    {
+      "completion_length": 449.0,
+      "epoch": 172.2,
+      "grad_norm": 0.5308093428611755,
+      "kl": 0.827573299407959,
+      "learning_rate": 2.603599250453349e-06,
+      "loss": 0.0331,
+      "reward": 3.1276378631591797,
+      "reward_std": 0.10545176267623901,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9285714626312256,
+      "rewards/wrapped_driving_reward": 0.32406648993492126,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 861
+    },
+    {
+      "completion_length": 452.0,
+      "epoch": 172.4,
+      "grad_norm": 0.6962711215019226,
+      "kl": 0.7359398603439331,
+      "learning_rate": 2.5981495393976718e-06,
+      "loss": 0.0294,
+      "reward": 2.441457748413086,
+      "reward_std": 2.2943224906921387,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.5585423111915588,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 862
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 172.6,
+      "grad_norm": 0.6115143895149231,
+      "kl": 1.3224482536315918,
+      "learning_rate": 2.592699361185002e-06,
+      "loss": 0.0529,
+      "reward": 1.3186888694763184,
+      "reward_std": 3.2175989151000977,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.0563111305236816,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 863
+    },
+    {
+      "completion_length": 421.0,
+      "epoch": 172.8,
+      "grad_norm": 0.648125410079956,
+      "kl": 1.027504563331604,
+      "learning_rate": 2.587248741756253e-06,
+      "loss": 0.0411,
+      "reward": -0.10881543159484863,
+      "reward_std": 1.7823691368103027,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -3.1088154315948486,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 864
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 173.0,
+      "grad_norm": 0.57094806432724,
+      "kl": 1.3319047689437866,
+      "learning_rate": 2.5817977070544408e-06,
+      "loss": 0.0533,
+      "reward": 3.846771717071533,
+      "reward_std": 0.01095542497932911,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8467714786529541,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 865
+    },
+    {
+      "completion_length": 329.0,
+      "epoch": 173.2,
+      "grad_norm": 0.808718740940094,
+      "kl": 0.6123311519622803,
+      "learning_rate": 2.5763462830245573e-06,
+      "loss": 0.0245,
+      "reward": 2.751035690307617,
+      "reward_std": 0.5787281394004822,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.12396407127380371,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 866
+    },
+    {
+      "completion_length": 385.0,
+      "epoch": 173.4,
+      "grad_norm": 0.6459723711013794,
+      "kl": 0.8475527763366699,
+      "learning_rate": 2.570894495613446e-06,
+      "loss": 0.0339,
+      "reward": 3.8003950119018555,
+      "reward_std": 0.08562152832746506,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8003950119018555,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 867
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 173.6,
+      "grad_norm": 0.600118100643158,
+      "kl": 1.1404991149902344,
+      "learning_rate": 2.5654423707696834e-06,
+      "loss": 0.0456,
+      "reward": 1.9638034105300903,
+      "reward_std": 0.40437883138656616,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8999999761581421,
+      "rewards/wrapped_driving_reward": -0.43619662523269653,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 868
+    },
+    {
+      "completion_length": 491.0,
+      "epoch": 173.8,
+      "grad_norm": 0.4509470462799072,
+      "kl": 0.9321443438529968,
+      "learning_rate": 2.5599899344434478e-06,
+      "loss": 0.0373,
+      "reward": 3.2264249324798584,
+      "reward_std": 0.32198405265808105,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9166666865348816,
+      "rewards/wrapped_driving_reward": 0.3097583055496216,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 869
+    },
+    {
+      "completion_length": 399.0,
+      "epoch": 174.0,
+      "grad_norm": 0.578890323638916,
+      "kl": 1.2454216480255127,
+      "learning_rate": 2.554537212586403e-06,
+      "loss": 0.0498,
+      "reward": 3.1222589015960693,
+      "reward_std": 0.37537509202957153,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.37225890159606934,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 870
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 174.2,
+      "grad_norm": 0.4751075804233551,
+      "kl": 1.4109488725662231,
+      "learning_rate": 2.5490842311515706e-06,
+      "loss": 0.0564,
+      "reward": 0.9386399388313293,
+      "reward_std": 2.473573684692383,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.9363600015640259,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 871
+    },
+    {
+      "completion_length": 474.0,
+      "epoch": 174.4,
+      "grad_norm": 0.5804294943809509,
+      "kl": 0.7709574103355408,
+      "learning_rate": 2.543631016093209e-06,
+      "loss": 0.0308,
+      "reward": 3.2371604442596436,
+      "reward_std": 0.5636550188064575,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.4871605336666107,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 872
+    },
+    {
+      "completion_length": 469.0,
+      "epoch": 174.6,
+      "grad_norm": 0.460746705532074,
+      "kl": 1.7947441339492798,
+      "learning_rate": 2.5381775933666865e-06,
+      "loss": 0.0718,
+      "reward": 2.5671794414520264,
+      "reward_std": 0.5235273241996765,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.1828204095363617,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 873
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 174.8,
+      "grad_norm": 0.5847729444503784,
+      "kl": 0.7998570799827576,
+      "learning_rate": 2.5327239889283613e-06,
+      "loss": 0.032,
+      "reward": 0.6826847195625305,
+      "reward_std": 3.0680971145629883,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.6923152208328247,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 874
+    },
+    {
+      "completion_length": 418.0,
+      "epoch": 175.0,
+      "grad_norm": 0.5057737231254578,
+      "kl": 0.4298799932003021,
+      "learning_rate": 2.527270228735456e-06,
+      "loss": 0.0172,
+      "reward": 3.286527156829834,
+      "reward_std": 0.6175230741500854,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.5365269780158997,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 875
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 175.2,
+      "grad_norm": 0.5232069492340088,
+      "kl": 0.9115723371505737,
+      "learning_rate": 2.521816338745935e-06,
+      "loss": 0.0365,
+      "reward": 1.682668924331665,
+      "reward_std": 3.1222028732299805,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.71875,
+      "rewards/wrapped_driving_reward": -0.786081075668335,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 876
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 175.4,
+      "grad_norm": 0.4667075574398041,
+      "kl": 1.3090119361877441,
+      "learning_rate": 2.5163623449183797e-06,
+      "loss": 0.0524,
+      "reward": 3.2737345695495605,
+      "reward_std": 0.5244059562683105,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.5237348079681396,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 877
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 175.6,
+      "grad_norm": 0.4833585023880005,
+      "kl": 1.0329177379608154,
+      "learning_rate": 2.510908273211867e-06,
+      "loss": 0.0413,
+      "reward": 1.6408182382583618,
+      "reward_std": 1.9210997819900513,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.96875,
+      "rewards/wrapped_driving_reward": -1.0779317617416382,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 878
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 175.8,
+      "grad_norm": 0.5182239413261414,
+      "kl": 0.934988796710968,
+      "learning_rate": 2.5054541495858427e-06,
+      "loss": 0.0374,
+      "reward": 0.9360474944114685,
+      "reward_std": 1.3490967750549316,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9564394354820251,
+      "rewards/wrapped_driving_reward": -1.7703919410705566,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 879
+    },
+    {
+      "completion_length": 475.0,
+      "epoch": 176.0,
+      "grad_norm": 0.5969507694244385,
+      "kl": 1.24098801612854,
+      "learning_rate": 2.5e-06,
+      "loss": 0.0496,
+      "reward": 1.8240338563919067,
+      "reward_std": 3.2637932300567627,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.4259660840034485,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 880
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 176.2,
+      "grad_norm": 0.537401020526886,
+      "kl": 0.6495235562324524,
+      "learning_rate": 2.494545850414158e-06,
+      "loss": 0.026,
+      "reward": 1.929699420928955,
+      "reward_std": 1.9812421798706055,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.6953005790710449,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 881
+    },
+    {
+      "completion_length": 386.0,
+      "epoch": 176.4,
+      "grad_norm": 0.5999033451080322,
+      "kl": 1.48947274684906,
+      "learning_rate": 2.489091726788134e-06,
+      "loss": 0.0596,
+      "reward": 1.8825178146362305,
+      "reward_std": 1.9242823123931885,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.1174821853637695,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 882
+    },
+    {
+      "completion_length": 428.0,
+      "epoch": 176.6,
+      "grad_norm": 0.5353307723999023,
+      "kl": 0.5975685715675354,
+      "learning_rate": 2.4836376550816207e-06,
+      "loss": 0.0239,
+      "reward": 0.9004546403884888,
+      "reward_std": 2.195589065551758,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9750000238418579,
+      "rewards/wrapped_driving_reward": -2.074545383453369,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 883
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 176.8,
+      "grad_norm": 0.6042259335517883,
+      "kl": 1.6270625591278076,
+      "learning_rate": 2.4781836612540656e-06,
+      "loss": 0.0651,
+      "reward": 1.2540318965911865,
+      "reward_std": 3.1778993606567383,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.625,
+      "rewards/wrapped_driving_reward": -0.9959681034088135,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 884
+    },
+    {
+      "completion_length": 429.0,
+      "epoch": 177.0,
+      "grad_norm": 0.5502638816833496,
+      "kl": 1.7028800249099731,
+      "learning_rate": 2.4727297712645446e-06,
+      "loss": 0.0681,
+      "reward": 3.253913640975952,
+      "reward_std": 0.4737393260002136,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.37891364097595215,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 885
+    },
+    {
+      "completion_length": 419.0,
+      "epoch": 177.2,
+      "grad_norm": 0.5808356404304504,
+      "kl": 0.5641009211540222,
+      "learning_rate": 2.4672760110716395e-06,
+      "loss": 0.0226,
+      "reward": 2.7367911338806152,
+      "reward_std": 0.09191377460956573,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.949999988079071,
+      "rewards/wrapped_driving_reward": -0.21320880949497223,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 886
+    },
+    {
+      "completion_length": 436.0,
+      "epoch": 177.4,
+      "grad_norm": 0.5215104222297668,
+      "kl": 1.0268008708953857,
+      "learning_rate": 2.4618224066333143e-06,
+      "loss": 0.0411,
+      "reward": 3.6914260387420654,
+      "reward_std": 0.2449469417333603,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8164260387420654,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 887
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 177.6,
+      "grad_norm": 0.5653296113014221,
+      "kl": 1.398972511291504,
+      "learning_rate": 2.4563689839067913e-06,
+      "loss": 0.056,
+      "reward": 2.985867738723755,
+      "reward_std": 0.318721204996109,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.23586782813072205,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 888
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 177.8,
+      "grad_norm": 0.6126723289489746,
+      "kl": 1.4683130979537964,
+      "learning_rate": 2.45091576884843e-06,
+      "loss": 0.0587,
+      "reward": 3.5237417221069336,
+      "reward_std": 0.20529741048812866,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9318181872367859,
+      "rewards/wrapped_driving_reward": 0.7169235348701477,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 889
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 178.0,
+      "grad_norm": 0.6861754655838013,
+      "kl": 0.6537904739379883,
+      "learning_rate": 2.4454627874135976e-06,
+      "loss": 0.0262,
+      "reward": -2.0917086601257324,
+      "reward_std": 1.6465706825256348,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -3.8417086601257324,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 890
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 178.2,
+      "grad_norm": 0.5680333971977234,
+      "kl": 1.307004451751709,
+      "learning_rate": 2.4400100655565535e-06,
+      "loss": 0.0523,
+      "reward": 1.553303599357605,
+      "reward_std": 3.3759968280792236,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.821696400642395,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 891
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 178.4,
+      "grad_norm": 0.5444093346595764,
+      "kl": 0.9408724308013916,
+      "learning_rate": 2.434557629230318e-06,
+      "loss": 0.0376,
+      "reward": 2.4606845378875732,
+      "reward_std": 0.2496892362833023,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.16431555151939392,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 892
+    },
+    {
+      "completion_length": 332.0,
+      "epoch": 178.6,
+      "grad_norm": 1.0570348501205444,
+      "kl": 1.3375152349472046,
+      "learning_rate": 2.4291055043865547e-06,
+      "loss": 0.0535,
+      "reward": 3.1871886253356934,
+      "reward_std": 0.2547098398208618,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.18718869984149933,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 893
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 178.8,
+      "grad_norm": 0.6612321734428406,
+      "kl": 0.9976045489311218,
+      "learning_rate": 2.423653716975444e-06,
+      "loss": 0.0399,
+      "reward": -1.1944348812103271,
+      "reward_std": 2.345881700515747,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -3.069434881210327,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 894
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 179.0,
+      "grad_norm": 0.5748969316482544,
+      "kl": 1.190211296081543,
+      "learning_rate": 2.41820229294556e-06,
+      "loss": 0.0476,
+      "reward": 3.7105839252471924,
+      "reward_std": 0.25292056798934937,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.875,
+      "rewards/wrapped_driving_reward": 0.8355838656425476,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 895
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 179.2,
+      "grad_norm": 0.44747933745384216,
+      "kl": 1.1943285465240479,
+      "learning_rate": 2.4127512582437486e-06,
+      "loss": 0.0478,
+      "reward": 3.0833992958068848,
+      "reward_std": 0.8473286628723145,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.45839935541152954,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 896
+    },
+    {
+      "completion_length": 485.0,
+      "epoch": 179.4,
+      "grad_norm": 0.48663514852523804,
+      "kl": 1.2799426317214966,
+      "learning_rate": 2.4073006388149992e-06,
+      "loss": 0.0512,
+      "reward": 3.4635372161865234,
+      "reward_std": 0.1930120885372162,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9642857313156128,
+      "rewards/wrapped_driving_reward": 0.4992515742778778,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 897
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 179.6,
+      "grad_norm": 0.5305557250976562,
+      "kl": 1.188493013381958,
+      "learning_rate": 2.4018504606023295e-06,
+      "loss": 0.0475,
+      "reward": -1.625,
+      "reward_std": 1.25,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 898
+    },
+    {
+      "completion_length": 343.0,
+      "epoch": 179.8,
+      "grad_norm": 0.5853403210639954,
+      "kl": 0.7435815334320068,
+      "learning_rate": 2.3964007495466523e-06,
+      "loss": 0.0297,
+      "reward": 2.5528595447540283,
+      "reward_std": 0.24696195125579834,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.4471404552459717,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 899
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 180.0,
+      "grad_norm": 0.5050544738769531,
+      "kl": 1.337159276008606,
+      "learning_rate": 2.3909515315866606e-06,
+      "loss": 0.0535,
+      "reward": 2.4426145553588867,
+      "reward_std": 0.22859933972358704,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.949999988079071,
+      "rewards/wrapped_driving_reward": -0.3823854923248291,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 900
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 180.2,
+      "grad_norm": 0.5133768320083618,
+      "kl": 0.31792768836021423,
+      "learning_rate": 2.385502832658699e-06,
+      "loss": 0.0127,
+      "reward": 3.5273141860961914,
+      "reward_std": 0.3408697247505188,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.7773142457008362,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 901
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 180.4,
+      "grad_norm": 0.7845637202262878,
+      "kl": 0.5812042355537415,
+      "learning_rate": 2.380054678696641e-06,
+      "loss": 0.0232,
+      "reward": 3.3489980697631836,
+      "reward_std": 0.296768456697464,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.4739980399608612,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 902
+    },
+    {
+      "completion_length": 464.0,
+      "epoch": 180.6,
+      "grad_norm": 0.05685199797153473,
+      "kl": 0.641217827796936,
+      "learning_rate": 2.3746070956317664e-06,
+      "loss": 0.0256,
+      "reward": -1.0,
+      "reward_std": 0.0,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 903
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 180.8,
+      "grad_norm": 0.611503005027771,
+      "kl": 1.1354436874389648,
+      "learning_rate": 2.3691601093926406e-06,
+      "loss": 0.0454,
+      "reward": 1.4279756546020508,
+      "reward_std": 3.2888336181640625,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9470243453979492,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 904
+    },
+    {
+      "completion_length": 410.0,
+      "epoch": 181.0,
+      "grad_norm": 0.5386159420013428,
+      "kl": 0.9454559683799744,
+      "learning_rate": 2.363713745904984e-06,
+      "loss": 0.0378,
+      "reward": 2.3741612434387207,
+      "reward_std": 0.2962803542613983,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.875,
+      "rewards/wrapped_driving_reward": -0.3758388161659241,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 905
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 181.2,
+      "grad_norm": 0.5813198089599609,
+      "kl": 0.8604588508605957,
+      "learning_rate": 2.358268031091556e-06,
+      "loss": 0.0344,
+      "reward": 3.2131857872009277,
+      "reward_std": 0.4338037073612213,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.4631856083869934,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 906
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 181.4,
+      "grad_norm": 0.565249502658844,
+      "kl": 1.1992684602737427,
+      "learning_rate": 2.3528229908720275e-06,
+      "loss": 0.048,
+      "reward": 2.822211980819702,
+      "reward_std": 0.4615172743797302,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.949999988079071,
+      "rewards/wrapped_driving_reward": -0.12778803706169128,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 907
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 181.6,
+      "grad_norm": 0.4963197112083435,
+      "kl": 1.3221276998519897,
+      "learning_rate": 2.3473786511628577e-06,
+      "loss": 0.0529,
+      "reward": 2.4108781814575195,
+      "reward_std": 0.7984676361083984,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.21412202715873718,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 908
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 181.8,
+      "grad_norm": 0.5683193802833557,
+      "kl": 1.0979485511779785,
+      "learning_rate": 2.3419350378771737e-06,
+      "loss": 0.0439,
+      "reward": 3.236751079559326,
+      "reward_std": 0.17679031193256378,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.23675096035003662,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 909
+    },
+    {
+      "completion_length": 488.0,
+      "epoch": 182.0,
+      "grad_norm": 0.6277251243591309,
+      "kl": 1.2162151336669922,
+      "learning_rate": 2.3364921769246423e-06,
+      "loss": 0.0486,
+      "reward": 3.5269453525543213,
+      "reward_std": 0.3045935332775116,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8678977489471436,
+      "rewards/wrapped_driving_reward": 0.784047544002533,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 910
+    },
+    {
+      "completion_length": 467.0,
+      "epoch": 182.2,
+      "grad_norm": 0.8996431827545166,
+      "kl": 1.1654489040374756,
+      "learning_rate": 2.3310500942113525e-06,
+      "loss": 0.0466,
+      "reward": 2.1091933250427246,
+      "reward_std": 3.406883955001831,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7222222089767456,
+      "rewards/wrapped_driving_reward": -0.36302876472473145,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 911
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 182.4,
+      "grad_norm": 0.458644837141037,
+      "kl": 1.0396286249160767,
+      "learning_rate": 2.325608815639687e-06,
+      "loss": 0.0416,
+      "reward": 1.601841926574707,
+      "reward_std": 1.9085595607757568,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9431818127632141,
+      "rewards/wrapped_driving_reward": -1.0913399457931519,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 912
+    },
+    {
+      "completion_length": 448.0,
+      "epoch": 182.6,
+      "grad_norm": 0.4919649064540863,
+      "kl": 0.8752197027206421,
+      "learning_rate": 2.3201683671082016e-06,
+      "loss": 0.035,
+      "reward": 0.7716760635375977,
+      "reward_std": 2.0968692302703857,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -2.2283239364624023,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 913
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 182.8,
+      "grad_norm": 0.6369486451148987,
+      "kl": 1.6084184646606445,
+      "learning_rate": 2.314728774511502e-06,
+      "loss": 0.0643,
+      "reward": 3.28770112991333,
+      "reward_std": 0.470788836479187,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.5377011299133301,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 914
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 183.0,
+      "grad_norm": 0.6774345636367798,
+      "kl": 0.25260379910469055,
+      "learning_rate": 2.3092900637401193e-06,
+      "loss": 0.0101,
+      "reward": 0.5662202835083008,
+      "reward_std": 2.430043935775757,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -2.058779716491699,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 915
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 183.2,
+      "grad_norm": 0.56723952293396,
+      "kl": 0.6391198039054871,
+      "learning_rate": 2.3038522606803882e-06,
+      "loss": 0.0256,
+      "reward": 1.4860867261886597,
+      "reward_std": 2.0119857788085938,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9375,
+      "rewards/wrapped_driving_reward": -1.2014132738113403,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 916
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 183.4,
+      "grad_norm": 0.5726743340492249,
+      "kl": 0.4170107841491699,
+      "learning_rate": 2.298415391214321e-06,
+      "loss": 0.0167,
+      "reward": 2.559278964996338,
+      "reward_std": 0.6913965940475464,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.19072100520133972,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 917
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 183.6,
+      "grad_norm": 0.563683032989502,
+      "kl": 0.6422572135925293,
+      "learning_rate": 2.29297948121949e-06,
+      "loss": 0.0257,
+      "reward": 3.118239641189575,
+      "reward_std": 0.38170966506004333,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9142857193946838,
+      "rewards/wrapped_driving_reward": 0.5789539217948914,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 918
+    },
+    {
+      "completion_length": 446.0,
+      "epoch": 183.8,
+      "grad_norm": 0.49984797835350037,
+      "kl": 0.9756030440330505,
+      "learning_rate": 2.287544556568896e-06,
+      "loss": 0.039,
+      "reward": 3.495962381362915,
+      "reward_std": 0.41120827198028564,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.620962381362915,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 919
+    },
+    {
+      "completion_length": 437.0,
+      "epoch": 184.0,
+      "grad_norm": 0.5067118406295776,
+      "kl": 1.6291453838348389,
+      "learning_rate": 2.2821106431308546e-06,
+      "loss": 0.0652,
+      "reward": 2.8344063758850098,
+      "reward_std": 0.3185327351093292,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.1655937284231186,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 920
+    },
+    {
+      "completion_length": 492.0,
+      "epoch": 184.2,
+      "grad_norm": 0.5153319239616394,
+      "kl": 0.6589206457138062,
+      "learning_rate": 2.276677766768866e-06,
+      "loss": 0.0264,
+      "reward": 2.9638633728027344,
+      "reward_std": 0.556221067905426,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.03613676130771637,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 921
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 184.4,
+      "grad_norm": 0.49304455518722534,
+      "kl": 0.7782519459724426,
+      "learning_rate": 2.271245953341494e-06,
+      "loss": 0.0311,
+      "reward": -0.006941735744476318,
+      "reward_std": 1.9861165285110474,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -3.006941795349121,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 922
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 184.6,
+      "grad_norm": 0.4311465919017792,
+      "kl": 1.5542595386505127,
+      "learning_rate": 2.265815228702245e-06,
+      "loss": 0.0622,
+      "reward": 2.506084442138672,
+      "reward_std": 2.3388657569885254,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.49391549825668335,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 923
+    },
+    {
+      "completion_length": 456.0,
+      "epoch": 184.8,
+      "grad_norm": 0.5129895806312561,
+      "kl": 1.0902389287948608,
+      "learning_rate": 2.26038561869944e-06,
+      "loss": 0.0436,
+      "reward": 3.8182454109191895,
+      "reward_std": 0.009648384526371956,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8182453513145447,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 924
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 185.0,
+      "grad_norm": 0.4159168004989624,
+      "kl": 1.704729437828064,
+      "learning_rate": 2.2549571491760985e-06,
+      "loss": 0.0682,
+      "reward": 0.9104196429252625,
+      "reward_std": 2.661571502685547,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.3395801782608032,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 925
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 185.2,
+      "grad_norm": 0.49064022302627563,
+      "kl": 1.2203823328018188,
+      "learning_rate": 2.24952984596981e-06,
+      "loss": 0.0488,
+      "reward": 2.0226497650146484,
+      "reward_std": 3.357693910598755,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.3523501753807068,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 926
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 185.4,
+      "grad_norm": 0.4919830858707428,
+      "kl": 0.5248955488204956,
+      "learning_rate": 2.2441037349126107e-06,
+      "loss": 0.021,
+      "reward": 0.7122844457626343,
+      "reward_std": 2.855849266052246,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.4127154350280762,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 927
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 185.6,
+      "grad_norm": 0.49590083956718445,
+      "kl": 1.2909214496612549,
+      "learning_rate": 2.238678841830867e-06,
+      "loss": 0.0516,
+      "reward": 2.668900489807129,
+      "reward_std": 0.48628270626068115,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9166666865348816,
+      "rewards/wrapped_driving_reward": -0.12276604771614075,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 928
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 185.8,
+      "grad_norm": 0.601020872592926,
+      "kl": 0.44025442004203796,
+      "learning_rate": 2.2332551925451436e-06,
+      "loss": 0.0176,
+      "reward": 1.2071342468261719,
+      "reward_std": 2.818470001220703,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.734375,
+      "rewards/wrapped_driving_reward": -1.1522407531738281,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 929
+    },
+    {
+      "completion_length": 415.0,
+      "epoch": 186.0,
+      "grad_norm": 0.6828198432922363,
+      "kl": 0.971450924873352,
+      "learning_rate": 2.2278328128700893e-06,
+      "loss": 0.0389,
+      "reward": 3.055501699447632,
+      "reward_std": 0.4391587972640991,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.180501788854599,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 930
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 186.2,
+      "grad_norm": 0.5190205574035645,
+      "kl": 0.7996862530708313,
+      "learning_rate": 2.2224117286143063e-06,
+      "loss": 0.032,
+      "reward": 0.7720038890838623,
+      "reward_std": 2.37174391746521,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.9779961109161377,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 931
+    },
+    {
+      "completion_length": 407.0,
+      "epoch": 186.4,
+      "grad_norm": 1.0009135007858276,
+      "kl": 1.1303850412368774,
+      "learning_rate": 2.2169919655802338e-06,
+      "loss": 0.0452,
+      "reward": 3.2563962936401367,
+      "reward_std": 0.2707050144672394,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.875,
+      "rewards/wrapped_driving_reward": 0.5063963532447815,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 932
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 186.6,
+      "grad_norm": 0.5876384973526001,
+      "kl": 1.2698439359664917,
+      "learning_rate": 2.2115735495640212e-06,
+      "loss": 0.0508,
+      "reward": 3.4274063110351562,
+      "reward_std": 0.5421848297119141,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.552406370639801,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 933
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 186.8,
+      "grad_norm": 0.433315247297287,
+      "kl": 1.0054932832717896,
+      "learning_rate": 2.2061565063554063e-06,
+      "loss": 0.0402,
+      "reward": 1.0803213119506836,
+      "reward_std": 3.504443883895874,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.169678807258606,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 934
+    },
+    {
+      "completion_length": 496.0,
+      "epoch": 187.0,
+      "grad_norm": 0.5682697892189026,
+      "kl": 1.0767149925231934,
+      "learning_rate": 2.2007408617375944e-06,
+      "loss": 0.0431,
+      "reward": 1.842423439025879,
+      "reward_std": 3.241006374359131,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.6575766801834106,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 935
+    },
+    {
+      "completion_length": 463.0,
+      "epoch": 187.2,
+      "grad_norm": 0.5375118255615234,
+      "kl": 1.372418999671936,
+      "learning_rate": 2.195326641487132e-06,
+      "loss": 0.0549,
+      "reward": 3.8171489238739014,
+      "reward_std": 0.022597435861825943,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9886363744735718,
+      "rewards/wrapped_driving_reward": 0.8285125494003296,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 936
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 187.4,
+      "grad_norm": 0.7156015634536743,
+      "kl": 1.1508220434188843,
+      "learning_rate": 2.1899138713737876e-06,
+      "loss": 0.046,
+      "reward": 1.4790055751800537,
+      "reward_std": 3.656684637069702,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.7709944844245911,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 937
+    },
+    {
+      "completion_length": 480.0,
+      "epoch": 187.6,
+      "grad_norm": 0.498759925365448,
+      "kl": 1.381080150604248,
+      "learning_rate": 2.1845025771604263e-06,
+      "loss": 0.0552,
+      "reward": 2.6462209224700928,
+      "reward_std": 0.09563220292329788,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9583333134651184,
+      "rewards/wrapped_driving_reward": -0.3121124505996704,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 938
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 187.8,
+      "grad_norm": 0.4997135102748871,
+      "kl": 1.664068579673767,
+      "learning_rate": 2.1790927846028894e-06,
+      "loss": 0.0666,
+      "reward": 3.414687156677246,
+      "reward_std": 0.5244473218917847,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9375,
+      "rewards/wrapped_driving_reward": 0.7271872162818909,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 939
+    },
+    {
+      "completion_length": 410.0,
+      "epoch": 188.0,
+      "grad_norm": 0.5040246248245239,
+      "kl": 0.9109085202217102,
+      "learning_rate": 2.173684519449872e-06,
+      "loss": 0.0364,
+      "reward": 1.0221105813980103,
+      "reward_std": 1.975825309753418,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.8528895378112793,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 940
+    },
+    {
+      "completion_length": 496.0,
+      "epoch": 188.2,
+      "grad_norm": 0.478110134601593,
+      "kl": 0.8791089057922363,
+      "learning_rate": 2.1682778074427955e-06,
+      "loss": 0.0352,
+      "reward": 2.912130832672119,
+      "reward_std": 0.4089687168598175,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.08786918222904205,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 941
+    },
+    {
+      "completion_length": 486.0,
+      "epoch": 188.4,
+      "grad_norm": 0.5197477340698242,
+      "kl": 0.9789789915084839,
+      "learning_rate": 2.1628726743156933e-06,
+      "loss": 0.0392,
+      "reward": 3.500166893005371,
+      "reward_std": 0.3842155933380127,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.5001667737960815,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 942
+    },
+    {
+      "completion_length": 464.0,
+      "epoch": 188.6,
+      "grad_norm": 0.47153791785240173,
+      "kl": 0.7001530528068542,
+      "learning_rate": 2.1574691457950805e-06,
+      "loss": 0.028,
+      "reward": 0.8245824575424194,
+      "reward_std": 2.4226925373077393,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -2.05041766166687,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 943
+    },
+    {
+      "completion_length": 338.0,
+      "epoch": 188.8,
+      "grad_norm": 0.910094678401947,
+      "kl": 1.3480033874511719,
+      "learning_rate": 2.1520672475998374e-06,
+      "loss": 0.0539,
+      "reward": 3.2739739418029785,
+      "reward_std": 0.31017762422561646,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.27397388219833374,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 944
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 189.0,
+      "grad_norm": 0.49829399585723877,
+      "kl": 1.7790124416351318,
+      "learning_rate": 2.146667005441082e-06,
+      "loss": 0.0712,
+      "reward": 2.5157852172851562,
+      "reward_std": 0.26337930560112,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.35921457409858704,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 945
+    },
+    {
+      "completion_length": 459.0,
+      "epoch": 189.2,
+      "grad_norm": 0.5464431047439575,
+      "kl": 0.9494650959968567,
+      "learning_rate": 2.1412684450220524e-06,
+      "loss": 0.038,
+      "reward": 3.2252697944641113,
+      "reward_std": 0.448541522026062,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9583333134651184,
+      "rewards/wrapped_driving_reward": 0.39193639159202576,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 946
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 189.4,
+      "grad_norm": 0.49399006366729736,
+      "kl": 1.0156264305114746,
+      "learning_rate": 2.1358715920379814e-06,
+      "loss": 0.0406,
+      "reward": 2.7399253845214844,
+      "reward_std": 0.6259793639183044,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.010074526071548462,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 947
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 189.6,
+      "grad_norm": 0.5094689130783081,
+      "kl": 1.19485604763031,
+      "learning_rate": 2.1304764721759736e-06,
+      "loss": 0.0478,
+      "reward": 1.7101205587387085,
+      "reward_std": 3.811704158782959,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.5398794412612915,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 948
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 189.8,
+      "grad_norm": 0.5310368537902832,
+      "kl": 0.5689219832420349,
+      "learning_rate": 2.1250831111148873e-06,
+      "loss": 0.0228,
+      "reward": -0.20518773794174194,
+      "reward_std": 4.116705417633057,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.8301877975463867,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 949
+    },
+    {
+      "completion_length": 459.0,
+      "epoch": 190.0,
+      "grad_norm": 0.46584364771842957,
+      "kl": 1.1638586521148682,
+      "learning_rate": 2.1196915345252085e-06,
+      "loss": 0.0466,
+      "reward": 2.8259994983673096,
+      "reward_std": 0.048405181616544724,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.17400041222572327,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 950
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 190.2,
+      "grad_norm": 0.561079740524292,
+      "kl": 1.2665772438049316,
+      "learning_rate": 2.114301768068928e-06,
+      "loss": 0.0507,
+      "reward": 0.7488883137702942,
+      "reward_std": 3.192291021347046,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.1261117458343506,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 951
+    },
+    {
+      "completion_length": 489.0,
+      "epoch": 190.4,
+      "grad_norm": 0.5475406646728516,
+      "kl": 0.964567244052887,
+      "learning_rate": 2.1089138373994226e-06,
+      "loss": 0.0386,
+      "reward": 3.7516729831695557,
+      "reward_std": 0.17056289315223694,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9166666865348816,
+      "rewards/wrapped_driving_reward": 0.8350062370300293,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 952
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 190.6,
+      "grad_norm": 0.44733574986457825,
+      "kl": 0.8066545128822327,
+      "learning_rate": 2.1035277681613325e-06,
+      "loss": 0.0323,
+      "reward": -0.9628837704658508,
+      "reward_std": 2.6176187992095947,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -3.087883949279785,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 953
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 190.8,
+      "grad_norm": 0.6097102165222168,
+      "kl": 0.7529609203338623,
+      "learning_rate": 2.0981435859904347e-06,
+      "loss": 0.0301,
+      "reward": 2.6820497512817383,
+      "reward_std": 0.7259109020233154,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.18204964697360992,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 954
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 191.0,
+      "grad_norm": 0.48875048756599426,
+      "kl": 0.9087252616882324,
+      "learning_rate": 2.0927613165135285e-06,
+      "loss": 0.0363,
+      "reward": 2.9861044883728027,
+      "reward_std": 0.2680176794528961,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.11110442876815796,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 955
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 191.2,
+      "grad_norm": 0.6656526327133179,
+      "kl": 1.1753857135772705,
+      "learning_rate": 2.087380985348306e-06,
+      "loss": 0.047,
+      "reward": 2.8272321224212646,
+      "reward_std": 0.5713879466056824,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.04776783287525177,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 956
+    },
+    {
+      "completion_length": 480.0,
+      "epoch": 191.4,
+      "grad_norm": 0.5438945293426514,
+      "kl": 0.9849552512168884,
+      "learning_rate": 2.0820026181032356e-06,
+      "loss": 0.0394,
+      "reward": 3.3579394817352295,
+      "reward_std": 0.3115105628967285,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.35793960094451904,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 957
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 191.6,
+      "grad_norm": 0.4994749128818512,
+      "kl": 1.893036961555481,
+      "learning_rate": 2.0766262403774388e-06,
+      "loss": 0.0757,
+      "reward": 3.585599899291992,
+      "reward_std": 0.2987547218799591,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.835599958896637,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 958
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 191.8,
+      "grad_norm": 0.44899803400039673,
+      "kl": 1.1986360549926758,
+      "learning_rate": 2.0712518777605646e-06,
+      "loss": 0.0479,
+      "reward": 2.7396349906921387,
+      "reward_std": 0.25676774978637695,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.2603647708892822,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 959
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 192.0,
+      "grad_norm": 0.7886030673980713,
+      "kl": 1.005682349205017,
+      "learning_rate": 2.0658795558326745e-06,
+      "loss": 0.0402,
+      "reward": 2.3123788833618164,
+      "reward_std": 0.9716589450836182,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.18762129545211792,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 960
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 192.2,
+      "grad_norm": 0.5696879029273987,
+      "kl": 1.0842654705047607,
+      "learning_rate": 2.0605093001641138e-06,
+      "loss": 0.0434,
+      "reward": -0.4991178512573242,
+      "reward_std": 3.465120553970337,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -2.249117851257324,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 961
+    },
+    {
+      "completion_length": 483.0,
+      "epoch": 192.4,
+      "grad_norm": 0.6350774168968201,
+      "kl": 1.3506046533584595,
+      "learning_rate": 2.0551411363153963e-06,
+      "loss": 0.054,
+      "reward": 3.539217948913574,
+      "reward_std": 0.3723929822444916,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.5392177700996399,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 962
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 192.6,
+      "grad_norm": 0.49636659026145935,
+      "kl": 1.1797510385513306,
+      "learning_rate": 2.0497750898370757e-06,
+      "loss": 0.0472,
+      "reward": 2.146454095840454,
+      "reward_std": 0.8986145257949829,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9583333134651184,
+      "rewards/wrapped_driving_reward": -0.5618792176246643,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 963
+    },
+    {
+      "completion_length": 418.0,
+      "epoch": 192.8,
+      "grad_norm": 0.7261518836021423,
+      "kl": 0.6447563171386719,
+      "learning_rate": 2.0444111862696313e-06,
+      "loss": 0.0258,
+      "reward": 3.6410467624664307,
+      "reward_std": 0.2871745228767395,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.7660467624664307,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 964
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 193.0,
+      "grad_norm": 0.6479581594467163,
+      "kl": 0.9154292941093445,
+      "learning_rate": 2.039049451143342e-06,
+      "loss": 0.0366,
+      "reward": 2.706965923309326,
+      "reward_std": 0.5069039463996887,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9545454382896423,
+      "rewards/wrapped_driving_reward": -0.12257963418960571,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 965
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 193.2,
+      "grad_norm": 0.5651826858520508,
+      "kl": 1.1386218070983887,
+      "learning_rate": 2.0336899099781636e-06,
+      "loss": 0.0455,
+      "reward": 2.9072165489196777,
+      "reward_std": 0.7617118954658508,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.15721642971038818,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 966
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 193.4,
+      "grad_norm": 0.4863123893737793,
+      "kl": 0.2626260221004486,
+      "learning_rate": 2.0283325882836126e-06,
+      "loss": 0.0105,
+      "reward": 1.4328651428222656,
+      "reward_std": 3.309224843978882,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9421347975730896,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 967
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 193.6,
+      "grad_norm": 0.4705544114112854,
+      "kl": 0.5639627575874329,
+      "learning_rate": 2.022977511558638e-06,
+      "loss": 0.0226,
+      "reward": 3.455763816833496,
+      "reward_std": 0.25063416361808777,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.875,
+      "rewards/wrapped_driving_reward": 0.8307638764381409,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 968
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 193.8,
+      "grad_norm": 0.460371196269989,
+      "kl": 1.470854640007019,
+      "learning_rate": 2.0176247052915078e-06,
+      "loss": 0.0588,
+      "reward": 2.5401976108551025,
+      "reward_std": 0.22898149490356445,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8854166865348816,
+      "rewards/wrapped_driving_reward": -0.2202192097902298,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 969
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 194.0,
+      "grad_norm": 0.49431416392326355,
+      "kl": 1.0209243297576904,
+      "learning_rate": 2.01227419495968e-06,
+      "loss": 0.0408,
+      "reward": 1.3964576721191406,
+      "reward_std": 3.288207530975342,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.6726190447807312,
+      "rewards/wrapped_driving_reward": -0.9011614322662354,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 970
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 194.2,
+      "grad_norm": 0.7239106893539429,
+      "kl": 0.4642423689365387,
+      "learning_rate": 2.0069260060296854e-06,
+      "loss": 0.0186,
+      "reward": 3.4259514808654785,
+      "reward_std": 0.2672251760959625,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9318181872367859,
+      "rewards/wrapped_driving_reward": 0.49413320422172546,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 971
+    },
+    {
+      "completion_length": 446.0,
+      "epoch": 194.4,
+      "grad_norm": 0.5294839143753052,
+      "kl": 0.9924250245094299,
+      "learning_rate": 2.0015801639570076e-06,
+      "loss": 0.0397,
+      "reward": 1.7371912002563477,
+      "reward_std": 1.832466959953308,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9107142686843872,
+      "rewards/wrapped_driving_reward": -1.04852294921875,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 972
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 194.6,
+      "grad_norm": 0.5141122341156006,
+      "kl": 1.1143356561660767,
+      "learning_rate": 1.996236694185957e-06,
+      "loss": 0.0446,
+      "reward": 1.409009575843811,
+      "reward_std": 3.6155614852905273,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7291666865348816,
+      "rewards/wrapped_driving_reward": -0.8201570510864258,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 973
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 194.8,
+      "grad_norm": 0.5137813687324524,
+      "kl": 0.6774077415466309,
+      "learning_rate": 1.9908956221495567e-06,
+      "loss": 0.0271,
+      "reward": 3.294393301010132,
+      "reward_std": 0.5990864634513855,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.96875,
+      "rewards/wrapped_driving_reward": 0.3256433606147766,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 974
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 195.0,
+      "grad_norm": 0.6752596497535706,
+      "kl": 0.8337000012397766,
+      "learning_rate": 1.985556973269413e-06,
+      "loss": 0.0333,
+      "reward": 1.436514139175415,
+      "reward_std": 3.2987518310546875,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9384859800338745,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 975
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 195.2,
+      "grad_norm": 0.6689338088035583,
+      "kl": 0.6801450848579407,
+      "learning_rate": 1.9802207729556023e-06,
+      "loss": 0.0272,
+      "reward": 3.2011795043945312,
+      "reward_std": 0.6917397975921631,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.5761793851852417,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 976
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 195.4,
+      "grad_norm": 0.560636579990387,
+      "kl": 0.8436287045478821,
+      "learning_rate": 1.9748870466065444e-06,
+      "loss": 0.0337,
+      "reward": -0.6823706030845642,
+      "reward_std": 2.8480265140533447,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -2.932370662689209,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 977
+    },
+    {
+      "completion_length": 406.0,
+      "epoch": 195.6,
+      "grad_norm": 0.5505325794219971,
+      "kl": 0.6946606040000916,
+      "learning_rate": 1.9695558196088846e-06,
+      "loss": 0.0278,
+      "reward": 3.748781204223633,
+      "reward_std": 0.14656925201416016,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.7487812042236328,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 978
+    },
+    {
+      "completion_length": 494.0,
+      "epoch": 195.8,
+      "grad_norm": 0.6658393144607544,
+      "kl": 1.4692611694335938,
+      "learning_rate": 1.964227117337374e-06,
+      "loss": 0.0588,
+      "reward": 2.6252613067626953,
+      "reward_std": 0.14504392445087433,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9791666865348816,
+      "rewards/wrapped_driving_reward": -0.35390543937683105,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 979
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 196.0,
+      "grad_norm": 0.5573218464851379,
+      "kl": 0.9373491406440735,
+      "learning_rate": 1.958900965154743e-06,
+      "loss": 0.0375,
+      "reward": 3.241145133972168,
+      "reward_std": 0.23681922256946564,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.949999988079071,
+      "rewards/wrapped_driving_reward": 0.29114508628845215,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 980
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 196.2,
+      "grad_norm": 0.5613752603530884,
+      "kl": 0.5691253542900085,
+      "learning_rate": 1.9535773884115894e-06,
+      "loss": 0.0228,
+      "reward": 2.539503335952759,
+      "reward_std": 0.3668363094329834,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.335496723651886,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 981
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 196.4,
+      "grad_norm": 0.5520111918449402,
+      "kl": 0.8869532942771912,
+      "learning_rate": 1.9482564124462478e-06,
+      "loss": 0.0355,
+      "reward": 1.5741268396377563,
+      "reward_std": 3.410243511199951,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.4258730709552765,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 982
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 196.6,
+      "grad_norm": 0.4561191201210022,
+      "kl": 1.3822728395462036,
+      "learning_rate": 1.942938062584678e-06,
+      "loss": 0.0553,
+      "reward": 2.657071113586426,
+      "reward_std": 0.630805253982544,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.3429288864135742,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 983
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 196.8,
+      "grad_norm": 102.46700286865234,
+      "kl": 10.124485969543457,
+      "learning_rate": 1.937622364140338e-06,
+      "loss": 0.405,
+      "reward": 1.304389238357544,
+      "reward_std": 3.5367956161499023,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.945610761642456,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 984
+    },
+    {
+      "completion_length": 468.0,
+      "epoch": 197.0,
+      "grad_norm": 0.5332741141319275,
+      "kl": 1.2333430051803589,
+      "learning_rate": 1.9323093424140673e-06,
+      "loss": 0.0493,
+      "reward": 3.2868287563323975,
+      "reward_std": 0.4407948851585388,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.4118286073207855,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 985
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 197.2,
+      "grad_norm": 2.2904465198516846,
+      "kl": 0.9502900242805481,
+      "learning_rate": 1.926999022693965e-06,
+      "loss": 0.038,
+      "reward": 2.5531535148620605,
+      "reward_std": 1.0269813537597656,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.053153496235609055,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 986
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 197.4,
+      "grad_norm": 0.4350486099720001,
+      "kl": 1.0802429914474487,
+      "learning_rate": 1.9216914302552693e-06,
+      "loss": 0.0432,
+      "reward": 3.8217079639434814,
+      "reward_std": 0.014752035960555077,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8217079043388367,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 987
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 197.6,
+      "grad_norm": 0.5089243054389954,
+      "kl": 0.6424485445022583,
+      "learning_rate": 1.9163865903602374e-06,
+      "loss": 0.0257,
+      "reward": 3.3335821628570557,
+      "reward_std": 0.20193266868591309,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.33358216285705566,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 988
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 197.8,
+      "grad_norm": 0.5370872616767883,
+      "kl": 0.7112429738044739,
+      "learning_rate": 1.9110845282580253e-06,
+      "loss": 0.0285,
+      "reward": 2.584986925125122,
+      "reward_std": 0.441521018743515,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.2900131940841675,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 989
+    },
+    {
+      "completion_length": 448.0,
+      "epoch": 198.0,
+      "grad_norm": 0.4503290355205536,
+      "kl": 0.8598915934562683,
+      "learning_rate": 1.9057852691845677e-06,
+      "loss": 0.0344,
+      "reward": 3.0328993797302246,
+      "reward_std": 0.5645407438278198,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.032899484038352966,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 990
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 198.2,
+      "grad_norm": 0.4580807089805603,
+      "kl": 1.1359678506851196,
+      "learning_rate": 1.9004888383624596e-06,
+      "loss": 0.0454,
+      "reward": 3.2506661415100098,
+      "reward_std": 0.22670303285121918,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.2506660521030426,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 991
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 198.4,
+      "grad_norm": 0.44507095217704773,
+      "kl": 1.269830346107483,
+      "learning_rate": 1.895195261000831e-06,
+      "loss": 0.0508,
+      "reward": -0.32085761427879333,
+      "reward_std": 2.576094150543213,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.625,
+      "rewards/wrapped_driving_reward": -2.445857524871826,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 992
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 198.6,
+      "grad_norm": 0.519321084022522,
+      "kl": 1.010072946548462,
+      "learning_rate": 1.8899045622952337e-06,
+      "loss": 0.0404,
+      "reward": 2.7637908458709717,
+      "reward_std": 0.07630608975887299,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9583333134651184,
+      "rewards/wrapped_driving_reward": -0.1945425271987915,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 993
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 198.8,
+      "grad_norm": 0.5224224925041199,
+      "kl": 1.6516927480697632,
+      "learning_rate": 1.8846167674275175e-06,
+      "loss": 0.0661,
+      "reward": 3.530534267425537,
+      "reward_std": 0.47409287095069885,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9375,
+      "rewards/wrapped_driving_reward": 0.5930341482162476,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 994
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 199.0,
+      "grad_norm": 0.6636751890182495,
+      "kl": 0.4684174656867981,
+      "learning_rate": 1.8793319015657091e-06,
+      "loss": 0.0187,
+      "reward": 1.255099892616272,
+      "reward_std": 3.1992766857147217,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.625,
+      "rewards/wrapped_driving_reward": -0.3699001669883728,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 995
+    },
+    {
+      "completion_length": 489.0,
+      "epoch": 199.2,
+      "grad_norm": 0.4272195100784302,
+      "kl": 1.0444061756134033,
+      "learning_rate": 1.874049989863896e-06,
+      "loss": 0.0418,
+      "reward": 3.1618120670318604,
+      "reward_std": 0.15273608267307281,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.2868121862411499,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 996
+    },
+    {
+      "completion_length": 415.0,
+      "epoch": 199.4,
+      "grad_norm": 0.4787036180496216,
+      "kl": 1.132507085800171,
+      "learning_rate": 1.8687710574621051e-06,
+      "loss": 0.0453,
+      "reward": 3.367159843444824,
+      "reward_std": 0.13511835038661957,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.36715978384017944,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 997
+    },
+    {
+      "completion_length": 391.0,
+      "epoch": 199.6,
+      "grad_norm": 0.556138277053833,
+      "kl": 1.152779221534729,
+      "learning_rate": 1.8634951294861809e-06,
+      "loss": 0.0461,
+      "reward": 3.3572402000427246,
+      "reward_std": 0.3249020576477051,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9583333134651184,
+      "rewards/wrapped_driving_reward": 0.3989068865776062,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 998
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 199.8,
+      "grad_norm": 0.4999628961086273,
+      "kl": 0.8470302820205688,
+      "learning_rate": 1.8582222310476699e-06,
+      "loss": 0.0339,
+      "reward": 1.7823580503463745,
+      "reward_std": 3.5314066410064697,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.5833333134651184,
+      "rewards/wrapped_driving_reward": -0.4259752333164215,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 999
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 200.0,
+      "grad_norm": 0.5336737632751465,
+      "kl": 0.8840068578720093,
+      "learning_rate": 1.852952387243698e-06,
+      "loss": 0.0354,
+      "reward": 0.5844348669052124,
+      "reward_std": 2.797942638397217,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.625,
+      "rewards/wrapped_driving_reward": -1.1655651330947876,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 1000
+    },
+    {
+      "completion_length": 395.0,
+      "epoch": 200.2,
+      "grad_norm": 0.5439149141311646,
+      "kl": 0.9171858429908752,
+      "learning_rate": 1.8476856231568529e-06,
+      "loss": 0.0367,
+      "reward": 2.7416954040527344,
+      "reward_std": 0.13379621505737305,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.25830450654029846,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1001
+    },
+    {
+      "completion_length": 423.0,
+      "epoch": 200.4,
+      "grad_norm": 0.5809133648872375,
+      "kl": 1.1701741218566895,
+      "learning_rate": 1.8424219638550631e-06,
+      "loss": 0.0468,
+      "reward": 3.192802667617798,
+      "reward_std": 0.42982155084609985,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.3178027868270874,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1002
+    },
+    {
+      "completion_length": 478.0,
+      "epoch": 200.6,
+      "grad_norm": 0.050054844468832016,
+      "kl": 1.0839431285858154,
+      "learning_rate": 1.8371614343914798e-06,
+      "loss": 0.0434,
+      "reward": -1.0,
+      "reward_std": 0.0,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1003
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 200.8,
+      "grad_norm": 0.45238450169563293,
+      "kl": 1.2362264394760132,
+      "learning_rate": 1.831904059804358e-06,
+      "loss": 0.0494,
+      "reward": -1.0083858966827393,
+      "reward_std": 2.43581485748291,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -3.2583858966827393,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1004
+    },
+    {
+      "completion_length": 404.0,
+      "epoch": 201.0,
+      "grad_norm": 0.7248693704605103,
+      "kl": 1.0576118230819702,
+      "learning_rate": 1.8266498651169352e-06,
+      "loss": 0.0423,
+      "reward": 3.5685184001922607,
+      "reward_std": 0.5119433999061584,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8185184001922607,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1005
+    },
+    {
+      "completion_length": 412.0,
+      "epoch": 201.2,
+      "grad_norm": 0.6140061616897583,
+      "kl": 1.2763816118240356,
+      "learning_rate": 1.8213988753373147e-06,
+      "loss": 0.0511,
+      "reward": 3.5662946701049805,
+      "reward_std": 0.4883527159690857,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8162946701049805,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1006
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 201.4,
+      "grad_norm": 0.6511586308479309,
+      "kl": 1.0021275281906128,
+      "learning_rate": 1.816151115458345e-06,
+      "loss": 0.0401,
+      "reward": 2.3032379150390625,
+      "reward_std": 0.7256336808204651,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.1967620849609375,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 1007
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 201.6,
+      "grad_norm": 0.5640247464179993,
+      "kl": 1.7142525911331177,
+      "learning_rate": 1.8109066104575023e-06,
+      "loss": 0.0686,
+      "reward": 3.1420655250549316,
+      "reward_std": 0.4741675555706024,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.875,
+      "rewards/wrapped_driving_reward": 0.5170655250549316,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1008
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 201.8,
+      "grad_norm": 0.6370381712913513,
+      "kl": 0.6361904144287109,
+      "learning_rate": 1.8056653852967699e-06,
+      "loss": 0.0254,
+      "reward": 2.998880386352539,
+      "reward_std": 0.29611650109291077,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.0011198222637176514,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1009
+    },
+    {
+      "completion_length": 356.0,
+      "epoch": 202.0,
+      "grad_norm": 0.5455551743507385,
+      "kl": 0.7856366634368896,
+      "learning_rate": 1.8004274649225201e-06,
+      "loss": 0.0314,
+      "reward": 2.6704788208007812,
+      "reward_std": 0.2127101719379425,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9166666865348816,
+      "rewards/wrapped_driving_reward": -0.24618779122829437,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1010
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 202.2,
+      "grad_norm": 0.5027799010276794,
+      "kl": 0.4933527708053589,
+      "learning_rate": 1.7951928742653968e-06,
+      "loss": 0.0197,
+      "reward": -0.6408627033233643,
+      "reward_std": 2.9250099658966064,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -2.8908627033233643,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1011
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 202.4,
+      "grad_norm": 0.5143634676933289,
+      "kl": 0.797701358795166,
+      "learning_rate": 1.7899616382401935e-06,
+      "loss": 0.0319,
+      "reward": 2.7905075550079346,
+      "reward_std": 0.7774209976196289,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9750000238418579,
+      "rewards/wrapped_driving_reward": 0.06550752371549606,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1012
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 202.6,
+      "grad_norm": 0.507008969783783,
+      "kl": 0.9522870182991028,
+      "learning_rate": 1.78473378174574e-06,
+      "loss": 0.0381,
+      "reward": 2.6869428157806396,
+      "reward_std": 0.19496779143810272,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8522727489471436,
+      "rewards/wrapped_driving_reward": -0.16533000767230988,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1013
+    },
+    {
+      "completion_length": 420.0,
+      "epoch": 202.8,
+      "grad_norm": 0.5470250248908997,
+      "kl": 0.9132187962532043,
+      "learning_rate": 1.779509329664777e-06,
+      "loss": 0.0365,
+      "reward": 3.6490588188171387,
+      "reward_std": 0.373779296875,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.7740588188171387,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1014
+    },
+    {
+      "completion_length": 433.0,
+      "epoch": 203.0,
+      "grad_norm": 0.6416342854499817,
+      "kl": 0.6618080735206604,
+      "learning_rate": 1.7742883068638447e-06,
+      "loss": 0.0265,
+      "reward": 3.000938892364502,
+      "reward_std": 0.6245017647743225,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9166666865348816,
+      "rewards/wrapped_driving_reward": 0.3342721462249756,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1015
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 203.2,
+      "grad_norm": 0.483258455991745,
+      "kl": 0.580714225769043,
+      "learning_rate": 1.7690707381931585e-06,
+      "loss": 0.0232,
+      "reward": 1.2770686149597168,
+      "reward_std": 3.1966874599456787,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.0979313850402832,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1016
+    },
+    {
+      "completion_length": 429.0,
+      "epoch": 203.4,
+      "grad_norm": 0.6593189835548401,
+      "kl": 0.5912168622016907,
+      "learning_rate": 1.7638566484864955e-06,
+      "loss": 0.0236,
+      "reward": 2.8950648307800293,
+      "reward_std": 0.35040944814682007,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.10493528842926025,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1017
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 203.6,
+      "grad_norm": 15.040224075317383,
+      "kl": 2.463568687438965,
+      "learning_rate": 1.758646062561073e-06,
+      "loss": 0.0985,
+      "reward": 3.42419171333313,
+      "reward_std": 0.42538735270500183,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8392857313156128,
+      "rewards/wrapped_driving_reward": 0.8349060416221619,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1018
+    },
+    {
+      "completion_length": 462.0,
+      "epoch": 203.8,
+      "grad_norm": 0.9762257933616638,
+      "kl": 1.1341408491134644,
+      "learning_rate": 1.7534390052174316e-06,
+      "loss": 0.0454,
+      "reward": 2.2631609439849854,
+      "reward_std": 0.42577096819877625,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9166666865348816,
+      "rewards/wrapped_driving_reward": -0.27850574254989624,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1019
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 204.0,
+      "grad_norm": 0.4897487759590149,
+      "kl": 0.3763349652290344,
+      "learning_rate": 1.7482355012393177e-06,
+      "loss": 0.0151,
+      "reward": 2.9682703018188477,
+      "reward_std": 0.6788708567619324,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9642857313156128,
+      "rewards/wrapped_driving_reward": 0.2539844512939453,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1020
+    },
+    {
+      "completion_length": 461.0,
+      "epoch": 204.2,
+      "grad_norm": 0.5621995329856873,
+      "kl": 1.2740334272384644,
+      "learning_rate": 1.743035575393564e-06,
+      "loss": 0.051,
+      "reward": 3.57611346244812,
+      "reward_std": 0.39653274416923523,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8989898562431335,
+      "rewards/wrapped_driving_reward": 0.6771236062049866,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1021
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 204.4,
+      "grad_norm": 0.5257546901702881,
+      "kl": 1.2898865938186646,
+      "learning_rate": 1.7378392524299732e-06,
+      "loss": 0.0516,
+      "reward": 3.228548288345337,
+      "reward_std": 0.22194702923297882,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.22854836285114288,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1022
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 204.6,
+      "grad_norm": 0.6876675486564636,
+      "kl": 0.45151054859161377,
+      "learning_rate": 1.7326465570812001e-06,
+      "loss": 0.0181,
+      "reward": 1.244504451751709,
+      "reward_std": 3.163768768310547,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.1304956674575806,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1023
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 204.8,
+      "grad_norm": 0.860268771648407,
+      "kl": 1.709954023361206,
+      "learning_rate": 1.7274575140626318e-06,
+      "loss": 0.0684,
+      "reward": 1.433963418006897,
+      "reward_std": 3.2965734004974365,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.6944444179534912,
+      "rewards/wrapped_driving_reward": -0.8854809999465942,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1024
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 205.0,
+      "grad_norm": 0.6399371027946472,
+      "kl": 0.6393185257911682,
+      "learning_rate": 1.722272148072273e-06,
+      "loss": 0.0256,
+      "reward": -0.4811551570892334,
+      "reward_std": 3.781942367553711,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -2.1061551570892334,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1025
+    },
+    {
+      "completion_length": 484.0,
+      "epoch": 205.2,
+      "grad_norm": 0.7000553607940674,
+      "kl": 1.0189485549926758,
+      "learning_rate": 1.7170904837906266e-06,
+      "loss": 0.0408,
+      "reward": 2.9407882690429688,
+      "reward_std": 0.41353824734687805,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.05921188369393349,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1026
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 205.4,
+      "grad_norm": 0.8835301995277405,
+      "kl": 0.7759029865264893,
+      "learning_rate": 1.7119125458805767e-06,
+      "loss": 0.031,
+      "reward": -0.5380096435546875,
+      "reward_std": 3.717663526535034,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -2.1630096435546875,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1027
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 205.6,
+      "grad_norm": 0.4821813106536865,
+      "kl": 1.0392839908599854,
+      "learning_rate": 1.7067383589872702e-06,
+      "loss": 0.0416,
+      "reward": 3.607910633087158,
+      "reward_std": 0.28117692470550537,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.7329106330871582,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1028
+    },
+    {
+      "completion_length": 459.0,
+      "epoch": 205.8,
+      "grad_norm": 0.6500245332717896,
+      "kl": 1.064603328704834,
+      "learning_rate": 1.701567947738002e-06,
+      "loss": 0.0426,
+      "reward": 3.507437229156494,
+      "reward_std": 0.24832651019096375,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9791666865348816,
+      "rewards/wrapped_driving_reward": 0.6532704830169678,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1029
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 206.0,
+      "grad_norm": 0.627190887928009,
+      "kl": 0.9105367064476013,
+      "learning_rate": 1.6964013367420967e-06,
+      "loss": 0.0364,
+      "reward": 1.7362329959869385,
+      "reward_std": 1.8606616258621216,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.138766884803772,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1030
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 206.2,
+      "grad_norm": 0.47151973843574524,
+      "kl": 1.4535021781921387,
+      "learning_rate": 1.6912385505907886e-06,
+      "loss": 0.0581,
+      "reward": 2.035980224609375,
+      "reward_std": 2.0374884605407715,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.7140198349952698,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1031
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 206.4,
+      "grad_norm": 0.5669636726379395,
+      "kl": 0.6194060444831848,
+      "learning_rate": 1.686079613857109e-06,
+      "loss": 0.0248,
+      "reward": 1.4100918769836426,
+      "reward_std": 3.29785418510437,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7045454382896423,
+      "rewards/wrapped_driving_reward": -0.7944536209106445,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1032
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 206.6,
+      "grad_norm": 0.4978317618370056,
+      "kl": 1.0943533182144165,
+      "learning_rate": 1.6809245510957667e-06,
+      "loss": 0.0438,
+      "reward": 3.2189345359802246,
+      "reward_std": 0.207293301820755,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9375,
+      "rewards/wrapped_driving_reward": 0.28143468499183655,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1033
+    },
+    {
+      "completion_length": 420.0,
+      "epoch": 206.8,
+      "grad_norm": 1.7581123113632202,
+      "kl": 1.1364126205444336,
+      "learning_rate": 1.6757733868430325e-06,
+      "loss": 0.0455,
+      "reward": 2.6228537559509277,
+      "reward_std": 0.47371524572372437,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.12714631855487823,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1034
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 207.0,
+      "grad_norm": 0.559068500995636,
+      "kl": 0.886647641658783,
+      "learning_rate": 1.6706261456166205e-06,
+      "loss": 0.0355,
+      "reward": 3.5276756286621094,
+      "reward_std": 0.3682084083557129,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.7776756286621094,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1035
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 207.2,
+      "grad_norm": 0.5195751786231995,
+      "kl": 0.8854998350143433,
+      "learning_rate": 1.665482851915573e-06,
+      "loss": 0.0354,
+      "reward": 3.260310173034668,
+      "reward_std": 0.3043051064014435,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.3853102922439575,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1036
+    },
+    {
+      "completion_length": 471.0,
+      "epoch": 207.4,
+      "grad_norm": 0.5296968221664429,
+      "kl": 1.0175193548202515,
+      "learning_rate": 1.6603435302201449e-06,
+      "loss": 0.0407,
+      "reward": 2.624685287475586,
+      "reward_std": 0.1497790664434433,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9583333134651184,
+      "rewards/wrapped_driving_reward": -0.33364787697792053,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1037
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 207.6,
+      "grad_norm": 0.6594138145446777,
+      "kl": 0.5428394079208374,
+      "learning_rate": 1.6552082049916826e-06,
+      "loss": 0.0217,
+      "reward": 2.9804773330688477,
+      "reward_std": 0.4763607084751129,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.875,
+      "rewards/wrapped_driving_reward": 0.23047754168510437,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1038
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 207.8,
+      "grad_norm": 0.5048743486404419,
+      "kl": 1.3772960901260376,
+      "learning_rate": 1.6500769006725142e-06,
+      "loss": 0.0551,
+      "reward": 1.9491024017333984,
+      "reward_std": 3.634000539779663,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.699999988079071,
+      "rewards/wrapped_driving_reward": -0.37589770555496216,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1039
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 208.0,
+      "grad_norm": 0.5116450786590576,
+      "kl": 1.0854933261871338,
+      "learning_rate": 1.6449496416858285e-06,
+      "loss": 0.0434,
+      "reward": 1.3824856281280518,
+      "reward_std": 3.2671401500701904,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.7425143122673035,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1040
+    },
+    {
+      "completion_length": 494.0,
+      "epoch": 208.2,
+      "grad_norm": 0.5801917910575867,
+      "kl": 1.0017632246017456,
+      "learning_rate": 1.6398264524355579e-06,
+      "loss": 0.0401,
+      "reward": 2.3593571186065674,
+      "reward_std": 2.2485415935516357,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9861111044883728,
+      "rewards/wrapped_driving_reward": -0.5017538070678711,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1041
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 208.4,
+      "grad_norm": 0.5255222320556641,
+      "kl": 1.4332096576690674,
+      "learning_rate": 1.634707357306267e-06,
+      "loss": 0.0573,
+      "reward": -1.375,
+      "reward_std": 0.4787135720252991,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1042
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 208.6,
+      "grad_norm": 0.4436381459236145,
+      "kl": 1.3197137117385864,
+      "learning_rate": 1.629592380663034e-06,
+      "loss": 0.0528,
+      "reward": 3.112481117248535,
+      "reward_std": 0.5794194936752319,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8229166865348816,
+      "rewards/wrapped_driving_reward": 0.5395643711090088,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1043
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 208.8,
+      "grad_norm": 0.6016329526901245,
+      "kl": 0.4816608428955078,
+      "learning_rate": 1.6244815468513315e-06,
+      "loss": 0.0193,
+      "reward": 1.0697249174118042,
+      "reward_std": 3.4002115726470947,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.675000011920929,
+      "rewards/wrapped_driving_reward": -1.1052751541137695,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1044
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 209.0,
+      "grad_norm": 0.7506645321846008,
+      "kl": 0.38417497277259827,
+      "learning_rate": 1.6193748801969164e-06,
+      "loss": 0.0154,
+      "reward": 2.8853096961975098,
+      "reward_std": 0.676288902759552,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.135309636592865,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1045
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 209.2,
+      "grad_norm": 0.5556249618530273,
+      "kl": 0.6534287333488464,
+      "learning_rate": 1.6142724050057104e-06,
+      "loss": 0.0261,
+      "reward": 1.5790209770202637,
+      "reward_std": 3.4119036197662354,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.5459789633750916,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1046
+    },
+    {
+      "completion_length": 495.0,
+      "epoch": 209.4,
+      "grad_norm": 0.4513692855834961,
+      "kl": 1.3434898853302002,
+      "learning_rate": 1.6091741455636855e-06,
+      "loss": 0.0537,
+      "reward": 2.676198959350586,
+      "reward_std": 0.13005176186561584,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.3238009214401245,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1047
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 209.6,
+      "grad_norm": 0.4508095383644104,
+      "kl": 0.5549758672714233,
+      "learning_rate": 1.6040801261367494e-06,
+      "loss": 0.0222,
+      "reward": 2.484206199645996,
+      "reward_std": 0.43169355392456055,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.26579374074935913,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1048
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 209.8,
+      "grad_norm": 0.5455639958381653,
+      "kl": 1.0955522060394287,
+      "learning_rate": 1.5989903709706261e-06,
+      "loss": 0.0438,
+      "reward": 1.6265478134155273,
+      "reward_std": 3.7807846069335938,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.37345218658447266,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 1049
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 210.0,
+      "grad_norm": 0.5302932262420654,
+      "kl": 0.5576118230819702,
+      "learning_rate": 1.5939049042907463e-06,
+      "loss": 0.0223,
+      "reward": 2.5832266807556152,
+      "reward_std": 0.2513408660888672,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.08322659879922867,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 1050
+    },
+    {
+      "completion_length": 476.0,
+      "epoch": 210.2,
+      "grad_norm": 0.5545305013656616,
+      "kl": 0.9898893237113953,
+      "learning_rate": 1.588823750302126e-06,
+      "loss": 0.0396,
+      "reward": 3.3521203994750977,
+      "reward_std": 0.4853040277957916,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.47712045907974243,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1051
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 210.4,
+      "grad_norm": 0.4663363993167877,
+      "kl": 0.5388195514678955,
+      "learning_rate": 1.583746933189257e-06,
+      "loss": 0.0216,
+      "reward": -1.75,
+      "reward_std": 1.1902379989624023,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1052
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 210.6,
+      "grad_norm": 0.4905710518360138,
+      "kl": 1.282116413116455,
+      "learning_rate": 1.5786744771159877e-06,
+      "loss": 0.0513,
+      "reward": 2.6815030574798584,
+      "reward_std": 0.2345251739025116,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9285714626312256,
+      "rewards/wrapped_driving_reward": -0.24706846475601196,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1053
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 210.8,
+      "grad_norm": 0.510164201259613,
+      "kl": 0.7608659267425537,
+      "learning_rate": 1.5736064062254094e-06,
+      "loss": 0.0304,
+      "reward": 2.8379886150360107,
+      "reward_std": 0.6815150380134583,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9772727489471436,
+      "rewards/wrapped_driving_reward": 0.2357158362865448,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1054
+    },
+    {
+      "completion_length": 424.0,
+      "epoch": 211.0,
+      "grad_norm": 0.525722861289978,
+      "kl": 0.8469008207321167,
+      "learning_rate": 1.5685427446397427e-06,
+      "loss": 0.0339,
+      "reward": 0.9203387498855591,
+      "reward_std": 1.7987178564071655,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -2.0796613693237305,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1055
+    },
+    {
+      "completion_length": 469.0,
+      "epoch": 211.2,
+      "grad_norm": 0.44910043478012085,
+      "kl": 1.3364444971084595,
+      "learning_rate": 1.56348351646022e-06,
+      "loss": 0.0535,
+      "reward": 3.721999168395996,
+      "reward_std": 0.1361294388771057,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8999999761581421,
+      "rewards/wrapped_driving_reward": 0.8219992518424988,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1056
+    },
+    {
+      "completion_length": 453.0,
+      "epoch": 211.4,
+      "grad_norm": 31.67371940612793,
+      "kl": 3.312755823135376,
+      "learning_rate": 1.5584287457669733e-06,
+      "loss": 0.1325,
+      "reward": 3.5078036785125732,
+      "reward_std": 0.2815183699131012,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.5078036785125732,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1057
+    },
+    {
+      "completion_length": 422.0,
+      "epoch": 211.6,
+      "grad_norm": 0.5062034130096436,
+      "kl": 1.2556145191192627,
+      "learning_rate": 1.5533784566189179e-06,
+      "loss": 0.0502,
+      "reward": 2.7967429161071777,
+      "reward_std": 0.2846265435218811,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.20325705409049988,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1058
+    },
+    {
+      "completion_length": 361.0,
+      "epoch": 211.8,
+      "grad_norm": 0.5981904864311218,
+      "kl": 1.1003021001815796,
+      "learning_rate": 1.5483326730536389e-06,
+      "loss": 0.044,
+      "reward": 1.189009428024292,
+      "reward_std": 2.049252510070801,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.685990571975708,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1059
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 212.0,
+      "grad_norm": 0.5221306085586548,
+      "kl": 0.3395875096321106,
+      "learning_rate": 1.5432914190872757e-06,
+      "loss": 0.0136,
+      "reward": 3.175121545791626,
+      "reward_std": 0.3137187659740448,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.3001215159893036,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1060
+    },
+    {
+      "completion_length": 415.0,
+      "epoch": 212.2,
+      "grad_norm": 0.5279333591461182,
+      "kl": 1.2710953950881958,
+      "learning_rate": 1.5382547187144098e-06,
+      "loss": 0.0508,
+      "reward": 2.3237576484680176,
+      "reward_std": 0.5837838053703308,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.42624253034591675,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1061
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 212.4,
+      "grad_norm": 0.4919413626194,
+      "kl": 0.588172197341919,
+      "learning_rate": 1.533222595907949e-06,
+      "loss": 0.0235,
+      "reward": 1.5189170837402344,
+      "reward_std": 3.385728597640991,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.7310829758644104,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1062
+    },
+    {
+      "completion_length": 449.0,
+      "epoch": 212.6,
+      "grad_norm": 0.57431560754776,
+      "kl": 0.7798376083374023,
+      "learning_rate": 1.528195074619011e-06,
+      "loss": 0.0312,
+      "reward": 2.9638447761535645,
+      "reward_std": 0.5483216047286987,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.4638449549674988,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 1063
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 212.8,
+      "grad_norm": 0.510163426399231,
+      "kl": 1.2339779138565063,
+      "learning_rate": 1.5231721787768162e-06,
+      "loss": 0.0494,
+      "reward": 2.899096965789795,
+      "reward_std": 0.41430801153182983,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.2740969955921173,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1064
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 213.0,
+      "grad_norm": 0.45320412516593933,
+      "kl": 0.46405914425849915,
+      "learning_rate": 1.5181539322885652e-06,
+      "loss": 0.0186,
+      "reward": 3.5779013633728027,
+      "reward_std": 0.5079466104507446,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.987500011920929,
+      "rewards/wrapped_driving_reward": 0.8404014110565186,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1065
+    },
+    {
+      "completion_length": 405.0,
+      "epoch": 213.2,
+      "grad_norm": 0.5608161091804504,
+      "kl": 0.8827903866767883,
+      "learning_rate": 1.5131403590393323e-06,
+      "loss": 0.0353,
+      "reward": 3.27620792388916,
+      "reward_std": 0.29736989736557007,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.27620792388916016,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1066
+    },
+    {
+      "completion_length": 499.0,
+      "epoch": 213.4,
+      "grad_norm": 0.9510046243667603,
+      "kl": 1.3333206176757812,
+      "learning_rate": 1.5081314828919474e-06,
+      "loss": 0.0533,
+      "reward": 1.4509761333465576,
+      "reward_std": 2.1432945728302,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.1740238666534424,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1067
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 213.6,
+      "grad_norm": 0.5165545344352722,
+      "kl": 0.6743512153625488,
+      "learning_rate": 1.5031273276868846e-06,
+      "loss": 0.027,
+      "reward": 1.9350465536117554,
+      "reward_std": 3.6254355907440186,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.4399535059928894,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1068
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 213.8,
+      "grad_norm": 0.5095003247261047,
+      "kl": 0.7119073271751404,
+      "learning_rate": 1.4981279172421481e-06,
+      "loss": 0.0285,
+      "reward": 3.2879233360290527,
+      "reward_std": 0.1447080671787262,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.4129234552383423,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1069
+    },
+    {
+      "completion_length": 476.0,
+      "epoch": 214.0,
+      "grad_norm": 0.49473994970321655,
+      "kl": 1.050492763519287,
+      "learning_rate": 1.4931332753531575e-06,
+      "loss": 0.042,
+      "reward": 2.3811073303222656,
+      "reward_std": 2.2642571926116943,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.6188925504684448,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1070
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 214.2,
+      "grad_norm": 0.4672090411186218,
+      "kl": 1.6573748588562012,
+      "learning_rate": 1.488143425792637e-06,
+      "loss": 0.0663,
+      "reward": 3.5188655853271484,
+      "reward_std": 0.26520654559135437,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.6438655853271484,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1071
+    },
+    {
+      "completion_length": 451.0,
+      "epoch": 214.4,
+      "grad_norm": 0.4720667004585266,
+      "kl": 1.113652229309082,
+      "learning_rate": 1.4831583923105e-06,
+      "loss": 0.0445,
+      "reward": 3.539783000946045,
+      "reward_std": 0.3191594183444977,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.6647829413414001,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1072
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 214.6,
+      "grad_norm": 0.4849921762943268,
+      "kl": 0.7481405735015869,
+      "learning_rate": 1.4781781986337384e-06,
+      "loss": 0.0299,
+      "reward": 2.252484083175659,
+      "reward_std": 0.2958456575870514,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.4975159466266632,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1073
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 214.8,
+      "grad_norm": 0.4499855041503906,
+      "kl": 1.4098292589187622,
+      "learning_rate": 1.4732028684663075e-06,
+      "loss": 0.0564,
+      "reward": 2.700578451156616,
+      "reward_std": 1.0104776620864868,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.1744215041399002,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1074
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 215.0,
+      "grad_norm": 0.47490978240966797,
+      "kl": 1.1678500175476074,
+      "learning_rate": 1.4682324254890135e-06,
+      "loss": 0.0467,
+      "reward": 3.0782628059387207,
+      "reward_std": 0.2107284814119339,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.07826292514801025,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1075
+    },
+    {
+      "completion_length": 489.0,
+      "epoch": 215.2,
+      "grad_norm": 0.5023165941238403,
+      "kl": 1.3258552551269531,
+      "learning_rate": 1.463266893359403e-06,
+      "loss": 0.053,
+      "reward": 3.1338119506835938,
+      "reward_std": 0.2880606949329376,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.13381201028823853,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1076
+    },
+    {
+      "completion_length": 499.0,
+      "epoch": 215.4,
+      "grad_norm": 0.4662926495075226,
+      "kl": 1.1612616777420044,
+      "learning_rate": 1.4583062957116467e-06,
+      "loss": 0.0465,
+      "reward": 2.1145243644714355,
+      "reward_std": 2.1981258392333984,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.635475754737854,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1077
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 215.6,
+      "grad_norm": 0.4910287857055664,
+      "kl": 1.1709198951721191,
+      "learning_rate": 1.4533506561564305e-06,
+      "loss": 0.0468,
+      "reward": 3.287445068359375,
+      "reward_std": 0.0355953611433506,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9750000238418579,
+      "rewards/wrapped_driving_reward": 0.3124452233314514,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1078
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 215.8,
+      "grad_norm": 0.4700184166431427,
+      "kl": 0.8096769452095032,
+      "learning_rate": 1.4483999982808407e-06,
+      "loss": 0.0324,
+      "reward": 1.1281371116638184,
+      "reward_std": 3.095395565032959,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.246862769126892,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1079
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 216.0,
+      "grad_norm": 0.4228314161300659,
+      "kl": 1.389911413192749,
+      "learning_rate": 1.443454345648252e-06,
+      "loss": 0.0556,
+      "reward": 3.6443586349487305,
+      "reward_std": 0.2442273646593094,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9583333134651184,
+      "rewards/wrapped_driving_reward": 0.8110253214836121,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1080
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 216.2,
+      "grad_norm": 0.5527031421661377,
+      "kl": 0.8247435092926025,
+      "learning_rate": 1.4385137217982178e-06,
+      "loss": 0.033,
+      "reward": 2.317556619644165,
+      "reward_std": 2.2192442417144775,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9642857313156128,
+      "rewards/wrapped_driving_reward": -0.5217289924621582,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1081
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 216.4,
+      "grad_norm": 0.4471375048160553,
+      "kl": 1.0145710706710815,
+      "learning_rate": 1.4335781502463554e-06,
+      "loss": 0.0406,
+      "reward": 1.5648467540740967,
+      "reward_std": 2.0248138904571533,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.4351531267166138,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1082
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 216.6,
+      "grad_norm": 0.5729674100875854,
+      "kl": 0.49795395135879517,
+      "learning_rate": 1.4286476544842327e-06,
+      "loss": 0.0199,
+      "reward": 1.3453325033187866,
+      "reward_std": 3.3262927532196045,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.7796674966812134,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1083
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 216.8,
+      "grad_norm": 0.5138728022575378,
+      "kl": 1.1334092617034912,
+      "learning_rate": 1.4237222579792618e-06,
+      "loss": 0.0453,
+      "reward": 2.6465728282928467,
+      "reward_std": 0.2937064468860626,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.35342714190483093,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1084
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 217.0,
+      "grad_norm": 0.4705502688884735,
+      "kl": 1.0180473327636719,
+      "learning_rate": 1.4188019841745842e-06,
+      "loss": 0.0407,
+      "reward": -1.3174629211425781,
+      "reward_std": 3.3035850524902344,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -2.942462921142578,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1085
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 217.2,
+      "grad_norm": 0.4954066574573517,
+      "kl": 1.5164530277252197,
+      "learning_rate": 1.4138868564889575e-06,
+      "loss": 0.0607,
+      "reward": 3.2143332958221436,
+      "reward_std": 0.2695990204811096,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.33933329582214355,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1086
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 217.4,
+      "grad_norm": 0.4756050407886505,
+      "kl": 0.9657862782478333,
+      "learning_rate": 1.4089768983166445e-06,
+      "loss": 0.0386,
+      "reward": 3.208871841430664,
+      "reward_std": 0.27547264099121094,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.3338716924190521,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1087
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 217.6,
+      "grad_norm": 0.534946084022522,
+      "kl": 0.6347689032554626,
+      "learning_rate": 1.4040721330273063e-06,
+      "loss": 0.0254,
+      "reward": 1.3529741764068604,
+      "reward_std": 3.5933613777160645,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7272727489471436,
+      "rewards/wrapped_driving_reward": -0.37429845333099365,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 1088
+    },
+    {
+      "completion_length": 460.0,
+      "epoch": 217.8,
+      "grad_norm": 0.5797332525253296,
+      "kl": 1.1790854930877686,
+      "learning_rate": 1.3991725839658855e-06,
+      "loss": 0.0472,
+      "reward": 3.780576467514038,
+      "reward_std": 0.14467857778072357,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.7805765271186829,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1089
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 218.0,
+      "grad_norm": 0.5514100193977356,
+      "kl": 0.5633188486099243,
+      "learning_rate": 1.3942782744524974e-06,
+      "loss": 0.0225,
+      "reward": 0.8650394678115845,
+      "reward_std": 2.9356367588043213,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.2599605321884155,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1090
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 218.2,
+      "grad_norm": 0.49869492650032043,
+      "kl": 0.6833009719848633,
+      "learning_rate": 1.3893892277823181e-06,
+      "loss": 0.0273,
+      "reward": 2.9415764808654785,
+      "reward_std": 0.5482802391052246,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.96875,
+      "rewards/wrapped_driving_reward": 0.34782645106315613,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1091
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 218.4,
+      "grad_norm": 0.43978843092918396,
+      "kl": 1.135629415512085,
+      "learning_rate": 1.3845054672254781e-06,
+      "loss": 0.0454,
+      "reward": -0.4088727831840515,
+      "reward_std": 2.2002780437469482,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -2.783872604370117,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 1092
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 218.6,
+      "grad_norm": 0.6305235624313354,
+      "kl": 0.9312289357185364,
+      "learning_rate": 1.379627016026944e-06,
+      "loss": 0.0372,
+      "reward": 2.9183311462402344,
+      "reward_std": 0.6726945042610168,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.16833138465881348,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1093
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 218.8,
+      "grad_norm": 0.5884646773338318,
+      "kl": 0.8569211363792419,
+      "learning_rate": 1.3747538974064122e-06,
+      "loss": 0.0343,
+      "reward": 3.333658456802368,
+      "reward_std": 0.42034944891929626,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.7086584568023682,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1094
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 219.0,
+      "grad_norm": 0.4997120201587677,
+      "kl": 0.8591310977935791,
+      "learning_rate": 1.369886134558201e-06,
+      "loss": 0.0344,
+      "reward": 2.594414234161377,
+      "reward_std": 0.260906457901001,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.15558591485023499,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1095
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 219.2,
+      "grad_norm": 0.6922900676727295,
+      "kl": 0.43327489495277405,
+      "learning_rate": 1.3650237506511333e-06,
+      "loss": 0.0173,
+      "reward": 1.6213303804397583,
+      "reward_std": 3.7782530784606934,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.3786695897579193,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 1096
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 219.4,
+      "grad_norm": 0.5262632966041565,
+      "kl": 0.4449964165687561,
+      "learning_rate": 1.3601667688284326e-06,
+      "loss": 0.0178,
+      "reward": 1.241672396659851,
+      "reward_std": 2.913693428039551,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.8833276629447937,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1097
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 219.6,
+      "grad_norm": 0.45210087299346924,
+      "kl": 0.9319807291030884,
+      "learning_rate": 1.355315212207608e-06,
+      "loss": 0.0373,
+      "reward": 2.6127288341522217,
+      "reward_std": 0.37591928243637085,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.26227113604545593,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1098
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 219.8,
+      "grad_norm": 0.5149539113044739,
+      "kl": 1.1235793828964233,
+      "learning_rate": 1.3504691038803504e-06,
+      "loss": 0.0449,
+      "reward": 1.5883556604385376,
+      "reward_std": 3.396470785140991,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.734375,
+      "rewards/wrapped_driving_reward": -0.7710193991661072,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1099
+    },
+    {
+      "completion_length": 418.0,
+      "epoch": 220.0,
+      "grad_norm": 0.5988738536834717,
+      "kl": 0.6427736878395081,
+      "learning_rate": 1.3456284669124159e-06,
+      "loss": 0.0257,
+      "reward": 2.4873180389404297,
+      "reward_std": 2.3253793716430664,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.5126819014549255,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1100
+    },
+    {
+      "completion_length": 377.0,
+      "epoch": 220.2,
+      "grad_norm": 0.5859512090682983,
+      "kl": 0.8706057667732239,
+      "learning_rate": 1.3407933243435175e-06,
+      "loss": 0.0348,
+      "reward": 2.5962235927581787,
+      "reward_std": 2.3989148139953613,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.40377646684646606,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1101
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 220.4,
+      "grad_norm": 0.42635753750801086,
+      "kl": 0.8328641057014465,
+      "learning_rate": 1.3359636991872215e-06,
+      "loss": 0.0333,
+      "reward": 2.9762845039367676,
+      "reward_std": 0.3018980324268341,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.10128439962863922,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1102
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 220.6,
+      "grad_norm": 0.48451152443885803,
+      "kl": 1.1016451120376587,
+      "learning_rate": 1.3311396144308297e-06,
+      "loss": 0.0441,
+      "reward": 3.3559389114379883,
+      "reward_std": 0.12013702094554901,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.3559388816356659,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1103
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 220.8,
+      "grad_norm": 0.6226598024368286,
+      "kl": 1.2990634441375732,
+      "learning_rate": 1.3263210930352737e-06,
+      "loss": 0.052,
+      "reward": 2.3522729873657227,
+      "reward_std": 0.4496834874153137,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9375,
+      "rewards/wrapped_driving_reward": -0.33522701263427734,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1104
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 221.0,
+      "grad_norm": 0.6005873084068298,
+      "kl": 0.7997850179672241,
+      "learning_rate": 1.3215081579350058e-06,
+      "loss": 0.032,
+      "reward": 3.7023112773895264,
+      "reward_std": 0.28063392639160156,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.827311098575592,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1105
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 221.2,
+      "grad_norm": 0.6006094217300415,
+      "kl": 1.6232116222381592,
+      "learning_rate": 1.3167008320378919e-06,
+      "loss": 0.0649,
+      "reward": 3.3058621883392334,
+      "reward_std": 0.25420600175857544,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.555862307548523,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1106
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 221.4,
+      "grad_norm": 0.4694947302341461,
+      "kl": 0.48686838150024414,
+      "learning_rate": 1.3118991382250968e-06,
+      "loss": 0.0195,
+      "reward": 0.3804311752319336,
+      "reward_std": 2.179936170578003,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -2.4945688247680664,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1107
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 221.6,
+      "grad_norm": 0.6662242412567139,
+      "kl": 0.7797883152961731,
+      "learning_rate": 1.307103099350979e-06,
+      "loss": 0.0312,
+      "reward": 1.4270281791687012,
+      "reward_std": 3.289504289627075,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.6979718208312988,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1108
+    },
+    {
+      "completion_length": 493.0,
+      "epoch": 221.8,
+      "grad_norm": 0.6694799661636353,
+      "kl": 0.9301208257675171,
+      "learning_rate": 1.3023127382429844e-06,
+      "loss": 0.0372,
+      "reward": 1.7880754470825195,
+      "reward_std": 2.5805933475494385,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9545454382896423,
+      "rewards/wrapped_driving_reward": -0.916469931602478,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1109
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 222.0,
+      "grad_norm": 0.4669792354106903,
+      "kl": 1.4729721546173096,
+      "learning_rate": 1.2975280777015315e-06,
+      "loss": 0.0589,
+      "reward": -0.10623854398727417,
+      "reward_std": 2.316403388977051,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8041666746139526,
+      "rewards/wrapped_driving_reward": -2.910405158996582,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1110
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 222.2,
+      "grad_norm": 0.5494128465652466,
+      "kl": 0.2666524350643158,
+      "learning_rate": 1.2927491404999077e-06,
+      "loss": 0.0107,
+      "reward": -0.36123692989349365,
+      "reward_std": 3.6370880603790283,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.8612369298934937,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 1111
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 222.4,
+      "grad_norm": 0.522520899772644,
+      "kl": 0.8005973100662231,
+      "learning_rate": 1.2879759493841577e-06,
+      "loss": 0.032,
+      "reward": -0.076751708984375,
+      "reward_std": 3.956779718399048,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.826751708984375,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1112
+    },
+    {
+      "completion_length": 387.0,
+      "epoch": 222.6,
+      "grad_norm": 0.5550836324691772,
+      "kl": 0.7252134680747986,
+      "learning_rate": 1.28320852707298e-06,
+      "loss": 0.029,
+      "reward": 3.7820980548858643,
+      "reward_std": 0.08571135252714157,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.7820981740951538,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1113
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 222.8,
+      "grad_norm": 0.4826516807079315,
+      "kl": 0.9718144536018372,
+      "learning_rate": 1.2784468962576136e-06,
+      "loss": 0.0389,
+      "reward": 2.5785608291625977,
+      "reward_std": 0.12967005372047424,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.96875,
+      "rewards/wrapped_driving_reward": -0.39018917083740234,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1114
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 223.0,
+      "grad_norm": 0.41361674666404724,
+      "kl": 1.2510775327682495,
+      "learning_rate": 1.2736910796017302e-06,
+      "loss": 0.05,
+      "reward": 2.613588809967041,
+      "reward_std": 0.9740625023841858,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.38641121983528137,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1115
+    },
+    {
+      "completion_length": 458.0,
+      "epoch": 223.2,
+      "grad_norm": 0.4774559736251831,
+      "kl": 1.3011237382888794,
+      "learning_rate": 1.2689410997413327e-06,
+      "loss": 0.052,
+      "reward": 1.3616100549697876,
+      "reward_std": 2.9077935218811035,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.1383899450302124,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1116
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 223.4,
+      "grad_norm": 0.40635526180267334,
+      "kl": 0.8614976406097412,
+      "learning_rate": 1.2641969792846393e-06,
+      "loss": 0.0345,
+      "reward": 3.288119316101074,
+      "reward_std": 0.03027961589396,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.28811922669410706,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1117
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 223.6,
+      "grad_norm": 0.42730268836021423,
+      "kl": 1.0905179977416992,
+      "learning_rate": 1.2594587408119804e-06,
+      "loss": 0.0436,
+      "reward": 3.6046500205993652,
+      "reward_std": 0.41561296582221985,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.7916666865348816,
+      "rewards/wrapped_driving_reward": 0.8129833340644836,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1118
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 223.8,
+      "grad_norm": 0.531570315361023,
+      "kl": 1.1119457483291626,
+      "learning_rate": 1.2547264068756899e-06,
+      "loss": 0.0445,
+      "reward": 1.88459050655365,
+      "reward_std": 1.0071583986282349,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8636363744735718,
+      "rewards/wrapped_driving_reward": -0.8540458083152771,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1119
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 224.0,
+      "grad_norm": 0.5223087072372437,
+      "kl": 1.3336859941482544,
+      "learning_rate": 1.2500000000000007e-06,
+      "loss": 0.0533,
+      "reward": 2.0367486476898193,
+      "reward_std": 2.0259768962860107,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9375,
+      "rewards/wrapped_driving_reward": -0.6507514119148254,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1120
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 224.2,
+      "grad_norm": 0.5794329643249512,
+      "kl": 0.8759583830833435,
+      "learning_rate": 1.2452795426809316e-06,
+      "loss": 0.035,
+      "reward": 3.3724653720855713,
+      "reward_std": 0.3152499198913574,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.37246543169021606,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1121
+    },
+    {
+      "completion_length": 494.0,
+      "epoch": 224.4,
+      "grad_norm": 0.44653820991516113,
+      "kl": 1.2264385223388672,
+      "learning_rate": 1.2405650573861846e-06,
+      "loss": 0.0491,
+      "reward": 2.5578770637512207,
+      "reward_std": 0.4199657738208771,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.31712275743484497,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1122
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 224.6,
+      "grad_norm": 0.6216111779212952,
+      "kl": 0.7895039319992065,
+      "learning_rate": 1.235856566555039e-06,
+      "loss": 0.0316,
+      "reward": 3.702702522277832,
+      "reward_std": 0.2452882081270218,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8277022838592529,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1123
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 224.8,
+      "grad_norm": 0.5253044366836548,
+      "kl": 0.7406594157218933,
+      "learning_rate": 1.2311540925982404e-06,
+      "loss": 0.0296,
+      "reward": 2.2451894283294678,
+      "reward_std": 0.7123855948448181,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.949999988079071,
+      "rewards/wrapped_driving_reward": -0.4548106789588928,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1124
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 225.0,
+      "grad_norm": 0.4567599594593048,
+      "kl": 0.5081362128257751,
+      "learning_rate": 1.2264576578978956e-06,
+      "loss": 0.0203,
+      "reward": 3.3672847747802734,
+      "reward_std": 0.10521022230386734,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.3672846555709839,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1125
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 225.2,
+      "grad_norm": 0.7277004718780518,
+      "kl": 0.6264821290969849,
+      "learning_rate": 1.2217672848073702e-06,
+      "loss": 0.0251,
+      "reward": 1.1245274543762207,
+      "reward_std": 3.1135880947113037,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.0004725456237793,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1126
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 225.4,
+      "grad_norm": 0.43354275822639465,
+      "kl": 1.6355204582214355,
+      "learning_rate": 1.217082995651175e-06,
+      "loss": 0.0654,
+      "reward": 3.3961920738220215,
+      "reward_std": 0.22789330780506134,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.5211922526359558,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1127
+    },
+    {
+      "completion_length": 289.0,
+      "epoch": 225.6,
+      "grad_norm": 0.7992777228355408,
+      "kl": 0.9495464563369751,
+      "learning_rate": 1.2124048127248644e-06,
+      "loss": 0.038,
+      "reward": 2.350940704345703,
+      "reward_std": 1.5608303546905518,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.3990592360496521,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1128
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 225.8,
+      "grad_norm": 0.5219174027442932,
+      "kl": 0.8311055898666382,
+      "learning_rate": 1.2077327582949313e-06,
+      "loss": 0.0332,
+      "reward": 0.8061639070510864,
+      "reward_std": 2.8907265663146973,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.318835973739624,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1129
+    },
+    {
+      "completion_length": 491.0,
+      "epoch": 226.0,
+      "grad_norm": 0.565902829170227,
+      "kl": 0.6430721282958984,
+      "learning_rate": 1.203066854598696e-06,
+      "loss": 0.0257,
+      "reward": 3.8361616134643555,
+      "reward_std": 0.028305277228355408,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8361614942550659,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1130
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 226.2,
+      "grad_norm": 0.49454358220100403,
+      "kl": 1.4771254062652588,
+      "learning_rate": 1.1984071238442065e-06,
+      "loss": 0.0591,
+      "reward": 3.726825714111328,
+      "reward_std": 0.16360139846801758,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9166666865348816,
+      "rewards/wrapped_driving_reward": 0.8101589679718018,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1131
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 226.4,
+      "grad_norm": 0.4966979920864105,
+      "kl": 0.693567156791687,
+      "learning_rate": 1.193753588210128e-06,
+      "loss": 0.0277,
+      "reward": 2.7115814685821533,
+      "reward_std": 0.711817741394043,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.16341853141784668,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1132
+    },
+    {
+      "completion_length": 493.0,
+      "epoch": 226.6,
+      "grad_norm": 0.42131781578063965,
+      "kl": 0.8615176677703857,
+      "learning_rate": 1.1891062698456383e-06,
+      "loss": 0.0345,
+      "reward": 2.31019926071167,
+      "reward_std": 2.2086000442504883,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9722222089767456,
+      "rewards/wrapped_driving_reward": -0.6620230078697205,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1133
+    },
+    {
+      "completion_length": 405.0,
+      "epoch": 226.8,
+      "grad_norm": 0.5451503396034241,
+      "kl": 1.1159465312957764,
+      "learning_rate": 1.1844651908703261e-06,
+      "loss": 0.0446,
+      "reward": 3.368283271789551,
+      "reward_std": 0.2502804100513458,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.36828312277793884,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1134
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 227.0,
+      "grad_norm": 0.4641178548336029,
+      "kl": 1.3155994415283203,
+      "learning_rate": 1.1798303733740801e-06,
+      "loss": 0.0526,
+      "reward": 2.527066230773926,
+      "reward_std": 0.3074857294559479,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9375,
+      "rewards/wrapped_driving_reward": -0.2854337990283966,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1135
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 227.2,
+      "grad_norm": 0.4675483703613281,
+      "kl": 1.2292439937591553,
+      "learning_rate": 1.1752018394169882e-06,
+      "loss": 0.0492,
+      "reward": 3.1753671169281006,
+      "reward_std": 0.5443548560142517,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9583333134651184,
+      "rewards/wrapped_driving_reward": 0.7170337438583374,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 1136
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 227.4,
+      "grad_norm": 0.4642682373523712,
+      "kl": 1.5533450841903687,
+      "learning_rate": 1.1705796110292287e-06,
+      "loss": 0.0621,
+      "reward": 2.701568365097046,
+      "reward_std": 0.4697892665863037,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.0484316349029541,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1137
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 227.6,
+      "grad_norm": 0.4544084668159485,
+      "kl": 1.6809604167938232,
+      "learning_rate": 1.1659637102109713e-06,
+      "loss": 0.0672,
+      "reward": 2.321577548980713,
+      "reward_std": 2.2529759407043457,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.42842233180999756,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1138
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 227.8,
+      "grad_norm": 0.4889006018638611,
+      "kl": 0.7387452125549316,
+      "learning_rate": 1.1613541589322658e-06,
+      "loss": 0.0295,
+      "reward": 1.773569941520691,
+      "reward_std": 1.8531171083450317,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.226430058479309,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1139
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 228.0,
+      "grad_norm": 0.5160104036331177,
+      "kl": 0.9329594969749451,
+      "learning_rate": 1.1567509791329402e-06,
+      "loss": 0.0373,
+      "reward": 3.1627237796783447,
+      "reward_std": 0.025614436715841293,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.2877238392829895,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1140
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 228.2,
+      "grad_norm": 0.5337912440299988,
+      "kl": 0.7600746750831604,
+      "learning_rate": 1.1521541927224994e-06,
+      "loss": 0.0304,
+      "reward": 3.1620514392852783,
+      "reward_std": 0.2856471538543701,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.2870514392852783,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1141
+    },
+    {
+      "completion_length": 349.0,
+      "epoch": 228.4,
+      "grad_norm": 0.6662641167640686,
+      "kl": 0.4100378453731537,
+      "learning_rate": 1.1475638215800155e-06,
+      "loss": 0.0164,
+      "reward": 2.713298797607422,
+      "reward_std": 0.07780962437391281,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.2867012917995453,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1142
+    },
+    {
+      "completion_length": 296.0,
+      "epoch": 228.6,
+      "grad_norm": 0.8487075567245483,
+      "kl": 0.8866725564002991,
+      "learning_rate": 1.1429798875540268e-06,
+      "loss": 0.0355,
+      "reward": 3.353952646255493,
+      "reward_std": 0.42592930793762207,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.3539527654647827,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1143
+    },
+    {
+      "completion_length": 455.0,
+      "epoch": 228.8,
+      "grad_norm": 0.5082492232322693,
+      "kl": 0.9051592946052551,
+      "learning_rate": 1.1384024124624324e-06,
+      "loss": 0.0362,
+      "reward": 2.104551315307617,
+      "reward_std": 3.4032726287841797,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.3954486846923828,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1144
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 229.0,
+      "grad_norm": 0.5631663799285889,
+      "kl": 0.3549773395061493,
+      "learning_rate": 1.1338314180923917e-06,
+      "loss": 0.0142,
+      "reward": 2.700592041015625,
+      "reward_std": 0.8660271763801575,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9583333134651184,
+      "rewards/wrapped_driving_reward": -0.13274119794368744,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1145
+    },
+    {
+      "completion_length": 413.0,
+      "epoch": 229.2,
+      "grad_norm": 0.5174888968467712,
+      "kl": 0.8225611448287964,
+      "learning_rate": 1.129266926200216e-06,
+      "loss": 0.0329,
+      "reward": 2.6267223358154297,
+      "reward_std": 0.2769395709037781,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.37327754497528076,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1146
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 229.4,
+      "grad_norm": 0.8155803680419922,
+      "kl": 0.4835788905620575,
+      "learning_rate": 1.1247089585112666e-06,
+      "loss": 0.0193,
+      "reward": 3.462742328643799,
+      "reward_std": 0.36503446102142334,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.46274232864379883,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1147
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 229.6,
+      "grad_norm": 0.6139227151870728,
+      "kl": 0.832255482673645,
+      "learning_rate": 1.1201575367198548e-06,
+      "loss": 0.0333,
+      "reward": 3.3516435623168945,
+      "reward_std": 0.41482865810394287,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.601643443107605,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1148
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 229.8,
+      "grad_norm": 0.4346941113471985,
+      "kl": 1.3379404544830322,
+      "learning_rate": 1.1156126824891327e-06,
+      "loss": 0.0535,
+      "reward": 3.501969337463379,
+      "reward_std": 0.2824113667011261,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.6269692778587341,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1149
+    },
+    {
+      "completion_length": 407.0,
+      "epoch": 230.0,
+      "grad_norm": 0.48849642276763916,
+      "kl": 0.7752863168716431,
+      "learning_rate": 1.1110744174509952e-06,
+      "loss": 0.031,
+      "reward": 1.6801931858062744,
+      "reward_std": 2.084578275680542,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.3198068141937256,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1150
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 230.2,
+      "grad_norm": 0.5179077982902527,
+      "kl": 0.2830272316932678,
+      "learning_rate": 1.1065427632059725e-06,
+      "loss": 0.0113,
+      "reward": 1.5219320058822632,
+      "reward_std": 3.3552567958831787,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.728067934513092,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1151
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 230.4,
+      "grad_norm": 0.5351471304893494,
+      "kl": 1.1633150577545166,
+      "learning_rate": 1.1020177413231334e-06,
+      "loss": 0.0465,
+      "reward": 3.370842695236206,
+      "reward_std": 0.3772900402545929,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.6208425760269165,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1152
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 230.6,
+      "grad_norm": 0.6424812078475952,
+      "kl": 0.8347950577735901,
+      "learning_rate": 1.0974993733399762e-06,
+      "loss": 0.0334,
+      "reward": 1.1425858736038208,
+      "reward_std": 3.4385924339294434,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.1074141263961792,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1153
+    },
+    {
+      "completion_length": 458.0,
+      "epoch": 230.8,
+      "grad_norm": 0.49495577812194824,
+      "kl": 1.0065813064575195,
+      "learning_rate": 1.0929876807623285e-06,
+      "loss": 0.0403,
+      "reward": 2.0097947120666504,
+      "reward_std": 3.3488450050354004,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.36520516872406006,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1154
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 231.0,
+      "grad_norm": 0.49082517623901367,
+      "kl": 1.4822145700454712,
+      "learning_rate": 1.0884826850642492e-06,
+      "loss": 0.0593,
+      "reward": 2.380051612854004,
+      "reward_std": 0.4968968331813812,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.36994829773902893,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1155
+    },
+    {
+      "completion_length": 384.0,
+      "epoch": 231.2,
+      "grad_norm": 0.7057926058769226,
+      "kl": 0.5097238421440125,
+      "learning_rate": 1.0839844076879186e-06,
+      "loss": 0.0204,
+      "reward": 3.1389360427856445,
+      "reward_std": 0.3136850893497467,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.13893592357635498,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1156
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 231.4,
+      "grad_norm": 0.4597265422344208,
+      "kl": 1.1270993947982788,
+      "learning_rate": 1.0794928700435402e-06,
+      "loss": 0.0451,
+      "reward": 3.4140944480895996,
+      "reward_std": 0.28165966272354126,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9272727370262146,
+      "rewards/wrapped_driving_reward": 0.48682159185409546,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1157
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 231.6,
+      "grad_norm": 0.47801825404167175,
+      "kl": 0.8727858066558838,
+      "learning_rate": 1.0750080935092425e-06,
+      "loss": 0.0349,
+      "reward": 0.22466373443603516,
+      "reward_std": 1.5351895093917847,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -2.525336265563965,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1158
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 231.8,
+      "grad_norm": 0.46797308325767517,
+      "kl": 0.9780285954475403,
+      "learning_rate": 1.0705300994309697e-06,
+      "loss": 0.0391,
+      "reward": 1.3459749221801758,
+      "reward_std": 2.8974549770355225,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.1540251970291138,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1159
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 232.0,
+      "grad_norm": 0.5172954797744751,
+      "kl": 0.7600094079971313,
+      "learning_rate": 1.0660589091223854e-06,
+      "loss": 0.0304,
+      "reward": 3.1870968341827393,
+      "reward_std": 0.4747726619243622,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9583333134651184,
+      "rewards/wrapped_driving_reward": 0.7287635803222656,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 1160
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 232.2,
+      "grad_norm": 0.5403032302856445,
+      "kl": 0.7036255598068237,
+      "learning_rate": 1.0615945438647687e-06,
+      "loss": 0.0281,
+      "reward": 1.9532389640808105,
+      "reward_std": 0.3156734108924866,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.6717610955238342,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1161
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 232.4,
+      "grad_norm": 0.6806216835975647,
+      "kl": 0.21848654747009277,
+      "learning_rate": 1.0571370249069163e-06,
+      "loss": 0.0087,
+      "reward": 2.9254488945007324,
+      "reward_std": 0.5159308314323425,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9772727489471436,
+      "rewards/wrapped_driving_reward": 0.07317619025707245,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1162
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 232.6,
+      "grad_norm": 0.5692815184593201,
+      "kl": 1.0410959720611572,
+      "learning_rate": 1.0526863734650365e-06,
+      "loss": 0.0416,
+      "reward": 0.7893268465995789,
+      "reward_std": 2.1887733936309814,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.949999988079071,
+      "rewards/wrapped_driving_reward": -2.160673141479492,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1163
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 232.8,
+      "grad_norm": 0.5501578450202942,
+      "kl": 0.7631838917732239,
+      "learning_rate": 1.0482426107226506e-06,
+      "loss": 0.0305,
+      "reward": 1.811545968055725,
+      "reward_std": 3.8753254413604736,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.4384539723396301,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1164
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 233.0,
+      "grad_norm": 0.6116912364959717,
+      "kl": 1.2737610340118408,
+      "learning_rate": 1.043805757830495e-06,
+      "loss": 0.051,
+      "reward": 2.938375949859619,
+      "reward_std": 0.8744444847106934,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.7916666865348816,
+      "rewards/wrapped_driving_reward": 0.5217090845108032,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1165
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 233.2,
+      "grad_norm": 0.558070182800293,
+      "kl": 1.3381707668304443,
+      "learning_rate": 1.0393758359064147e-06,
+      "loss": 0.0535,
+      "reward": 1.7771408557891846,
+      "reward_std": 3.851565361022949,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.47285914421081543,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1166
+    },
+    {
+      "completion_length": 495.0,
+      "epoch": 233.4,
+      "grad_norm": 0.5340891480445862,
+      "kl": 1.1490249633789062,
+      "learning_rate": 1.0349528660352664e-06,
+      "loss": 0.046,
+      "reward": 3.2934093475341797,
+      "reward_std": 0.7054101228713989,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.875,
+      "rewards/wrapped_driving_reward": 0.6684093475341797,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1167
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 233.6,
+      "grad_norm": 0.5877658128738403,
+      "kl": 1.257413387298584,
+      "learning_rate": 1.0305368692688175e-06,
+      "loss": 0.0503,
+      "reward": 2.490118980407715,
+      "reward_std": 0.4648132026195526,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.25988101959228516,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1168
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 233.8,
+      "grad_norm": 0.5253468751907349,
+      "kl": 1.0515938997268677,
+      "learning_rate": 1.0261278666256476e-06,
+      "loss": 0.0421,
+      "reward": 3.175070285797119,
+      "reward_std": 0.2983999252319336,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.17507022619247437,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1169
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 234.0,
+      "grad_norm": 0.4924323260784149,
+      "kl": 1.2200512886047363,
+      "learning_rate": 1.0217258790910447e-06,
+      "loss": 0.0488,
+      "reward": 2.797452688217163,
+      "reward_std": 0.508575975894928,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9642857313156128,
+      "rewards/wrapped_driving_reward": -0.04183301329612732,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1170
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 234.2,
+      "grad_norm": 0.6675472259521484,
+      "kl": 1.8396894931793213,
+      "learning_rate": 1.0173309276169075e-06,
+      "loss": 0.0736,
+      "reward": 3.0372731685638428,
+      "reward_std": 0.47866132855415344,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.16227304935455322,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1171
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 234.4,
+      "grad_norm": 0.4768546223640442,
+      "kl": 0.6031727194786072,
+      "learning_rate": 1.0129430331216473e-06,
+      "loss": 0.0241,
+      "reward": 2.268564462661743,
+      "reward_std": 2.1812455654144287,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.7314354777336121,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1172
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 234.6,
+      "grad_norm": 0.49328091740608215,
+      "kl": 0.5479208827018738,
+      "learning_rate": 1.0085622164900839e-06,
+      "loss": 0.0219,
+      "reward": 2.811215877532959,
+      "reward_std": 0.8501439094543457,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.1862158477306366,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1173
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 234.8,
+      "grad_norm": 0.5433307886123657,
+      "kl": 1.2493828535079956,
+      "learning_rate": 1.0041884985733524e-06,
+      "loss": 0.05,
+      "reward": 3.584805965423584,
+      "reward_std": 0.5257366299629211,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8348059058189392,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1174
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 235.0,
+      "grad_norm": 0.42357319593429565,
+      "kl": 0.8284562230110168,
+      "learning_rate": 9.99821900188798e-07,
+      "loss": 0.0331,
+      "reward": 2.7132484912872314,
+      "reward_std": 0.17555521428585052,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.28675156831741333,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1175
+    },
+    {
+      "completion_length": 491.0,
+      "epoch": 235.2,
+      "grad_norm": 0.4581468105316162,
+      "kl": 1.0952410697937012,
+      "learning_rate": 9.95462442119879e-07,
+      "loss": 0.0438,
+      "reward": 3.17767333984375,
+      "reward_std": 0.7674687504768372,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": 0.4276735186576843,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1176
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 235.4,
+      "grad_norm": 0.5303056836128235,
+      "kl": 0.9397453665733337,
+      "learning_rate": 9.911101451160714e-07,
+      "loss": 0.0376,
+      "reward": 1.899448037147522,
+      "reward_std": 1.95945143699646,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.100551962852478,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1177
+    },
+    {
+      "completion_length": 489.0,
+      "epoch": 235.6,
+      "grad_norm": 0.48008403182029724,
+      "kl": 1.350096344947815,
+      "learning_rate": 9.867650298927644e-07,
+      "loss": 0.054,
+      "reward": 3.18483304977417,
+      "reward_std": 0.3124329149723053,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9750000238418579,
+      "rewards/wrapped_driving_reward": 0.20983323454856873,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1178
+    },
+    {
+      "completion_length": 488.0,
+      "epoch": 235.8,
+      "grad_norm": 0.4235706329345703,
+      "kl": 0.7750853300094604,
+      "learning_rate": 9.824271171311638e-07,
+      "loss": 0.031,
+      "reward": 3.1104965209960938,
+      "reward_std": 0.3107686936855316,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.23549672961235046,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1179
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 236.0,
+      "grad_norm": 0.5463101267814636,
+      "kl": 1.1244868040084839,
+      "learning_rate": 9.780964274781984e-07,
+      "loss": 0.045,
+      "reward": 2.637910842895508,
+      "reward_std": 0.9993806481361389,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.23708899319171906,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1180
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 236.2,
+      "grad_norm": 0.933049201965332,
+      "kl": 1.3480939865112305,
+      "learning_rate": 9.737729815464132e-07,
+      "loss": 0.0539,
+      "reward": 2.018947124481201,
+      "reward_std": 1.1340445280075073,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9583333134651184,
+      "rewards/wrapped_driving_reward": -0.6893861889839172,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1181
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 236.4,
+      "grad_norm": 0.7749969959259033,
+      "kl": 1.3750650882720947,
+      "learning_rate": 9.694567999138764e-07,
+      "loss": 0.055,
+      "reward": 3.11362361907959,
+      "reward_std": 0.3091113865375519,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9226190447807312,
+      "rewards/wrapped_driving_reward": 0.5660045742988586,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1182
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 236.6,
+      "grad_norm": 0.574827253818512,
+      "kl": 0.5101785063743591,
+      "learning_rate": 9.651479031240837e-07,
+      "loss": 0.0204,
+      "reward": 3.0051515102386475,
+      "reward_std": 0.2615595757961273,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.2551514208316803,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1183
+    },
+    {
+      "completion_length": 464.0,
+      "epoch": 236.8,
+      "grad_norm": 0.5087046027183533,
+      "kl": 0.7921606302261353,
+      "learning_rate": 9.608463116858544e-07,
+      "loss": 0.0317,
+      "reward": 3.790410041809082,
+      "reward_std": 0.06367364525794983,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9642857313156128,
+      "rewards/wrapped_driving_reward": 0.8261243104934692,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1184
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 237.0,
+      "grad_norm": 0.5802391767501831,
+      "kl": 0.3770102858543396,
+      "learning_rate": 9.56552046073238e-07,
+      "loss": 0.0151,
+      "reward": 1.4945333003997803,
+      "reward_std": 3.66304612159729,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.7554667592048645,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1185
+    },
+    {
+      "completion_length": 490.0,
+      "epoch": 237.2,
+      "grad_norm": 0.4884765148162842,
+      "kl": 1.0093648433685303,
+      "learning_rate": 9.522651267254149e-07,
+      "loss": 0.0404,
+      "reward": 2.556617021560669,
+      "reward_std": 0.14493608474731445,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.44338300824165344,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1186
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 237.4,
+      "grad_norm": 0.5490802526473999,
+      "kl": 0.9102400541305542,
+      "learning_rate": 9.479855740466021e-07,
+      "loss": 0.0364,
+      "reward": 3.3429524898529053,
+      "reward_std": 0.5776273012161255,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8429526686668396,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 1187
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 237.6,
+      "grad_norm": 0.5087721943855286,
+      "kl": 0.569882869720459,
+      "learning_rate": 9.437134084059516e-07,
+      "loss": 0.0228,
+      "reward": 2.252133369445801,
+      "reward_std": 0.8320027589797974,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.949999988079071,
+      "rewards/wrapped_driving_reward": -0.5728667378425598,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1188
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 237.8,
+      "grad_norm": 0.5068728923797607,
+      "kl": 1.5162400007247925,
+      "learning_rate": 9.394486501374556e-07,
+      "loss": 0.0606,
+      "reward": 3.1553940773010254,
+      "reward_std": 0.48256558179855347,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.4053940773010254,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1189
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 238.0,
+      "grad_norm": 0.5118811726570129,
+      "kl": 0.21179969608783722,
+      "learning_rate": 9.351913195398523e-07,
+      "loss": 0.0085,
+      "reward": 2.5188398361206055,
+      "reward_std": 0.4586102068424225,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.3561601936817169,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1190
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 238.2,
+      "grad_norm": 0.5666443705558777,
+      "kl": 0.6030561923980713,
+      "learning_rate": 9.309414368765249e-07,
+      "loss": 0.0241,
+      "reward": 1.622738242149353,
+      "reward_std": 2.114074945449829,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.1272616386413574,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1191
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 238.4,
+      "grad_norm": 0.47993332147598267,
+      "kl": 0.8498132824897766,
+      "learning_rate": 9.266990223754069e-07,
+      "loss": 0.034,
+      "reward": 1.2041720151901245,
+      "reward_std": 3.1476194858551025,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.170827865600586,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1192
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 238.6,
+      "grad_norm": 0.46787920594215393,
+      "kl": 0.7299878001213074,
+      "learning_rate": 9.224640962288858e-07,
+      "loss": 0.0292,
+      "reward": 2.902759552001953,
+      "reward_std": 0.6612705588340759,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9642857313156128,
+      "rewards/wrapped_driving_reward": 0.18847376108169556,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1193
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 238.8,
+      "grad_norm": 0.4974771738052368,
+      "kl": 1.1346635818481445,
+      "learning_rate": 9.1823667859371e-07,
+      "loss": 0.0454,
+      "reward": 1.8710272312164307,
+      "reward_std": 3.914024591445923,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.3789728283882141,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1194
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 239.0,
+      "grad_norm": 0.46684443950653076,
+      "kl": 1.396405816078186,
+      "learning_rate": 9.140167895908867e-07,
+      "loss": 0.0559,
+      "reward": 3.314075469970703,
+      "reward_std": 0.4227040708065033,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.4390754699707031,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1195
+    },
+    {
+      "completion_length": 445.0,
+      "epoch": 239.2,
+      "grad_norm": 0.4115247130393982,
+      "kl": 1.4511141777038574,
+      "learning_rate": 9.098044493055899e-07,
+      "loss": 0.058,
+      "reward": 2.5170016288757324,
+      "reward_std": 0.2016606330871582,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.23299835622310638,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1196
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 239.4,
+      "grad_norm": 0.593860387802124,
+      "kl": 0.7119840383529663,
+      "learning_rate": 9.055996777870671e-07,
+      "loss": 0.0285,
+      "reward": 3.0464751720428467,
+      "reward_std": 0.1603722721338272,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.04647527635097504,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1197
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 239.6,
+      "grad_norm": 0.5286566615104675,
+      "kl": 0.9879803657531738,
+      "learning_rate": 9.014024950485384e-07,
+      "loss": 0.0395,
+      "reward": -0.24626564979553223,
+      "reward_std": 2.9094369411468506,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -2.6212656497955322,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1198
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 239.8,
+      "grad_norm": 0.7916061878204346,
+      "kl": 0.6191636323928833,
+      "learning_rate": 8.972129210671048e-07,
+      "loss": 0.0248,
+      "reward": 3.5133743286132812,
+      "reward_std": 0.47732067108154297,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.949999988079071,
+      "rewards/wrapped_driving_reward": 0.8133742213249207,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1199
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 240.0,
+      "grad_norm": 0.47770625352859497,
+      "kl": 1.2510874271392822,
+      "learning_rate": 8.930309757836517e-07,
+      "loss": 0.05,
+      "reward": 3.599067449569702,
+      "reward_std": 0.3348093330860138,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.5990675687789917,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1200
+    },
+    {
+      "completion_length": 478.0,
+      "epoch": 240.2,
+      "grad_norm": 0.46874821186065674,
+      "kl": 0.9536527395248413,
+      "learning_rate": 8.88856679102757e-07,
+      "loss": 0.0381,
+      "reward": 3.0143890380859375,
+      "reward_std": 0.35738250613212585,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.96875,
+      "rewards/wrapped_driving_reward": 0.045638855546712875,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1201
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 240.4,
+      "grad_norm": 0.6761062145233154,
+      "kl": 1.013494849205017,
+      "learning_rate": 8.846900508925909e-07,
+      "loss": 0.0405,
+      "reward": 2.766080856323242,
+      "reward_std": 0.42325618863105774,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.01608075574040413,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1202
+    },
+    {
+      "completion_length": 467.0,
+      "epoch": 240.6,
+      "grad_norm": 0.4714386463165283,
+      "kl": 1.0624254941940308,
+      "learning_rate": 8.805311109848255e-07,
+      "loss": 0.0425,
+      "reward": 0.04115086793899536,
+      "reward_std": 1.210576057434082,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -2.8338491916656494,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1203
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 240.8,
+      "grad_norm": 0.5720903873443604,
+      "kl": 0.9610505700111389,
+      "learning_rate": 8.763798791745413e-07,
+      "loss": 0.0384,
+      "reward": 0.7863779664039612,
+      "reward_std": 3.3226327896118164,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.5886220932006836,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1204
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 241.0,
+      "grad_norm": 0.4739234447479248,
+      "kl": 1.0068539381027222,
+      "learning_rate": 8.722363752201277e-07,
+      "loss": 0.0403,
+      "reward": 3.2589173316955566,
+      "reward_std": 0.43970987200737,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.5089173316955566,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1205
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 241.2,
+      "grad_norm": 0.7321770787239075,
+      "kl": 1.382409930229187,
+      "learning_rate": 8.681006188431945e-07,
+      "loss": 0.0553,
+      "reward": 2.1936376094818115,
+      "reward_std": 2.1304426193237305,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9166666865348816,
+      "rewards/wrapped_driving_reward": -0.5980291366577148,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1206
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 241.4,
+      "grad_norm": 0.5233954787254333,
+      "kl": 1.344728946685791,
+      "learning_rate": 8.639726297284742e-07,
+      "loss": 0.0538,
+      "reward": 2.238985538482666,
+      "reward_std": 0.4995039701461792,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.5110143423080444,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1207
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 241.6,
+      "grad_norm": 0.47989678382873535,
+      "kl": 1.4580020904541016,
+      "learning_rate": 8.598524275237321e-07,
+      "loss": 0.0583,
+      "reward": 3.4892873764038086,
+      "reward_std": 0.3079678416252136,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9222222566604614,
+      "rewards/wrapped_driving_reward": 0.6920652389526367,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1208
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 241.8,
+      "grad_norm": 0.5273704528808594,
+      "kl": 0.6805389523506165,
+      "learning_rate": 8.557400318396689e-07,
+      "loss": 0.0272,
+      "reward": 2.5610032081604004,
+      "reward_std": 0.5161240696907043,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9642857313156128,
+      "rewards/wrapped_driving_reward": -0.15328270196914673,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1209
+    },
+    {
+      "completion_length": 484.0,
+      "epoch": 242.0,
+      "grad_norm": 0.5125021934509277,
+      "kl": 0.8629850745201111,
+      "learning_rate": 8.516354622498279e-07,
+      "loss": 0.0345,
+      "reward": 1.4039452075958252,
+      "reward_std": 2.070845603942871,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.5960547924041748,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1210
+    },
+    {
+      "completion_length": 493.0,
+      "epoch": 242.2,
+      "grad_norm": 0.45823541283607483,
+      "kl": 0.7455308437347412,
+      "learning_rate": 8.475387382905065e-07,
+      "loss": 0.0298,
+      "reward": 3.006836414337158,
+      "reward_std": 0.4716048240661621,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9772727489471436,
+      "rewards/wrapped_driving_reward": 0.029563650488853455,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1211
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 242.4,
+      "grad_norm": 0.4908481240272522,
+      "kl": 1.2421576976776123,
+      "learning_rate": 8.434498794606569e-07,
+      "loss": 0.0497,
+      "reward": 0.8657845258712769,
+      "reward_std": 2.91644287109375,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.625,
+      "rewards/wrapped_driving_reward": -1.1342154741287231,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1212
+    },
+    {
+      "completion_length": 355.0,
+      "epoch": 242.6,
+      "grad_norm": 0.5992693305015564,
+      "kl": 1.011739730834961,
+      "learning_rate": 8.393689052217966e-07,
+      "loss": 0.0405,
+      "reward": 2.7241406440734863,
+      "reward_std": 0.37728339433670044,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.15085934102535248,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1213
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 242.8,
+      "grad_norm": 0.4369671940803528,
+      "kl": 1.1751753091812134,
+      "learning_rate": 8.352958349979146e-07,
+      "loss": 0.047,
+      "reward": 3.300814390182495,
+      "reward_std": 0.3936246335506439,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8999999761581421,
+      "rewards/wrapped_driving_reward": 0.5258143544197083,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1214
+    },
+    {
+      "completion_length": 476.0,
+      "epoch": 243.0,
+      "grad_norm": 0.49410322308540344,
+      "kl": 1.5363539457321167,
+      "learning_rate": 8.31230688175382e-07,
+      "loss": 0.0615,
+      "reward": 2.948258399963379,
+      "reward_std": 0.816761314868927,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.32325828075408936,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1215
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 243.2,
+      "grad_norm": 0.6465588212013245,
+      "kl": 0.921891987323761,
+      "learning_rate": 8.271734841028553e-07,
+      "loss": 0.0369,
+      "reward": 3.303527593612671,
+      "reward_std": 0.48781248927116394,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9444444179534912,
+      "rewards/wrapped_driving_reward": 0.7340831160545349,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1216
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 243.4,
+      "grad_norm": 0.461028516292572,
+      "kl": 1.6636199951171875,
+      "learning_rate": 8.231242420911856e-07,
+      "loss": 0.0665,
+      "reward": 2.9803595542907715,
+      "reward_std": 0.35423579812049866,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.10535944998264313,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1217
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 243.6,
+      "grad_norm": 0.4841845631599426,
+      "kl": 0.9123335480690002,
+      "learning_rate": 8.190829814133294e-07,
+      "loss": 0.0365,
+      "reward": 1.2024567127227783,
+      "reward_std": 3.1508467197418213,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.1725432872772217,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1218
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 243.8,
+      "grad_norm": 0.47493094205856323,
+      "kl": 0.5931532382965088,
+      "learning_rate": 8.150497213042552e-07,
+      "loss": 0.0237,
+      "reward": 3.621209144592285,
+      "reward_std": 0.2615833878517151,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9285714626312256,
+      "rewards/wrapped_driving_reward": 0.8176377415657043,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1219
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 244.0,
+      "grad_norm": 0.48062634468078613,
+      "kl": 0.858769953250885,
+      "learning_rate": 8.110244809608494e-07,
+      "loss": 0.0344,
+      "reward": 2.9860706329345703,
+      "reward_std": 0.5294856429100037,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.851190447807312,
+      "rewards/wrapped_driving_reward": 0.25988009572029114,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1220
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 244.2,
+      "grad_norm": 0.5263796448707581,
+      "kl": 0.6497914791107178,
+      "learning_rate": 8.070072795418277e-07,
+      "loss": 0.026,
+      "reward": 3.635040283203125,
+      "reward_std": 0.3345676064491272,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.7600401639938354,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1221
+    },
+    {
+      "completion_length": 435.0,
+      "epoch": 244.4,
+      "grad_norm": 0.5445246696472168,
+      "kl": 1.2088878154754639,
+      "learning_rate": 8.029981361676456e-07,
+      "loss": 0.0484,
+      "reward": 1.907090425491333,
+      "reward_std": 1.997957468032837,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.717909574508667,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1222
+    },
+    {
+      "completion_length": 453.0,
+      "epoch": 244.6,
+      "grad_norm": 0.48463916778564453,
+      "kl": 1.3090330362319946,
+      "learning_rate": 7.989970699204023e-07,
+      "loss": 0.0524,
+      "reward": 0.41780906915664673,
+      "reward_std": 1.9112588167190552,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9642857313156128,
+      "rewards/wrapped_driving_reward": -2.4214766025543213,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1223
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 244.8,
+      "grad_norm": 0.641181230545044,
+      "kl": 0.2546052038669586,
+      "learning_rate": 7.950040998437541e-07,
+      "loss": 0.0102,
+      "reward": 2.6896231174468994,
+      "reward_std": 0.31650683283805847,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.3103768229484558,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1224
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 245.0,
+      "grad_norm": 0.4748181104660034,
+      "kl": 1.1997054815292358,
+      "learning_rate": 7.910192449428216e-07,
+      "loss": 0.048,
+      "reward": 3.268188953399658,
+      "reward_std": 0.20170165598392487,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.26818886399269104,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1225
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 245.2,
+      "grad_norm": 0.623795211315155,
+      "kl": 1.1316560506820679,
+      "learning_rate": 7.87042524184102e-07,
+      "loss": 0.0453,
+      "reward": 3.1947622299194336,
+      "reward_std": 0.592230498790741,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.5697622895240784,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1226
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 245.4,
+      "grad_norm": 0.7470730543136597,
+      "kl": 0.4966695308685303,
+      "learning_rate": 7.830739564953754e-07,
+      "loss": 0.0199,
+      "reward": 0.9667633175849915,
+      "reward_std": 3.3536312580108643,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.6875,
+      "rewards/wrapped_driving_reward": -0.8457366824150085,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 1227
+    },
+    {
+      "completion_length": 418.0,
+      "epoch": 245.6,
+      "grad_norm": 0.502226710319519,
+      "kl": 1.3163583278656006,
+      "learning_rate": 7.791135607656147e-07,
+      "loss": 0.0527,
+      "reward": 3.8181052207946777,
+      "reward_std": 0.011925111524760723,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8181050419807434,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1228
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 245.8,
+      "grad_norm": 0.4864324927330017,
+      "kl": 0.38086503744125366,
+      "learning_rate": 7.751613558449009e-07,
+      "loss": 0.0152,
+      "reward": 2.6670265197753906,
+      "reward_std": 0.7196469902992249,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.20797355473041534,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1229
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 246.0,
+      "grad_norm": 0.5538790822029114,
+      "kl": 1.042360544204712,
+      "learning_rate": 7.712173605443269e-07,
+      "loss": 0.0417,
+      "reward": 1.4860494136810303,
+      "reward_std": 3.3296165466308594,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.7639504671096802,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1230
+    },
+    {
+      "completion_length": 326.0,
+      "epoch": 246.2,
+      "grad_norm": 0.5952561497688293,
+      "kl": 1.1502676010131836,
+      "learning_rate": 7.672815936359107e-07,
+      "loss": 0.046,
+      "reward": 3.453122138977051,
+      "reward_std": 0.46179068088531494,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.45312225818634033,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1231
+    },
+    {
+      "completion_length": 383.0,
+      "epoch": 246.4,
+      "grad_norm": 0.5355944633483887,
+      "kl": 1.0198066234588623,
+      "learning_rate": 7.633540738525066e-07,
+      "loss": 0.0408,
+      "reward": 2.422050714492798,
+      "reward_std": 0.48824384808540344,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8392857313156128,
+      "rewards/wrapped_driving_reward": -0.2922350764274597,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1232
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 246.6,
+      "grad_norm": 0.5583437085151672,
+      "kl": 1.425034761428833,
+      "learning_rate": 7.594348198877172e-07,
+      "loss": 0.057,
+      "reward": 3.0253515243530273,
+      "reward_std": 0.5947969555854797,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.2753515839576721,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1233
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 246.8,
+      "grad_norm": 0.505789041519165,
+      "kl": 0.7424576878547668,
+      "learning_rate": 7.555238503958001e-07,
+      "loss": 0.0297,
+      "reward": 0.5255041122436523,
+      "reward_std": 3.0668632984161377,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.7244958877563477,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1234
+    },
+    {
+      "completion_length": 373.0,
+      "epoch": 247.0,
+      "grad_norm": 0.6453691720962524,
+      "kl": 1.2416671514511108,
+      "learning_rate": 7.516211839915821e-07,
+      "loss": 0.0497,
+      "reward": 3.3668127059936523,
+      "reward_std": 0.0763755813241005,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9166666865348816,
+      "rewards/wrapped_driving_reward": 0.8251459002494812,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1235
+    },
+    {
+      "completion_length": 422.0,
+      "epoch": 247.2,
+      "grad_norm": 0.5285044312477112,
+      "kl": 0.7820354104042053,
+      "learning_rate": 7.477268392503728e-07,
+      "loss": 0.0313,
+      "reward": 2.843021869659424,
+      "reward_std": 0.12420294433832169,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.15697821974754333,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1236
+    },
+    {
+      "completion_length": 367.0,
+      "epoch": 247.4,
+      "grad_norm": 0.5692311525344849,
+      "kl": 1.0247697830200195,
+      "learning_rate": 7.43840834707871e-07,
+      "loss": 0.041,
+      "reward": 3.2826223373413086,
+      "reward_std": 0.19391310214996338,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.949999988079071,
+      "rewards/wrapped_driving_reward": 0.3326222598552704,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1237
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 247.6,
+      "grad_norm": 0.5409315228462219,
+      "kl": 0.7465785145759583,
+      "learning_rate": 7.399631888600797e-07,
+      "loss": 0.0299,
+      "reward": 1.5760520696640015,
+      "reward_std": 1.7244821786880493,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.2989479303359985,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1238
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 247.8,
+      "grad_norm": 0.4636867046356201,
+      "kl": 1.7611838579177856,
+      "learning_rate": 7.360939201632172e-07,
+      "loss": 0.0704,
+      "reward": 0.5179848670959473,
+      "reward_std": 3.3753364086151123,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.6499999761581421,
+      "rewards/wrapped_driving_reward": -1.7570151090621948,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1239
+    },
+    {
+      "completion_length": 415.0,
+      "epoch": 248.0,
+      "grad_norm": 0.6235267519950867,
+      "kl": 0.8498090505599976,
+      "learning_rate": 7.322330470336314e-07,
+      "loss": 0.034,
+      "reward": 0.9399834871292114,
+      "reward_std": 2.2655322551727295,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9166666865348816,
+      "rewards/wrapped_driving_reward": -1.8516831398010254,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1240
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 248.2,
+      "grad_norm": 0.5786277055740356,
+      "kl": 1.0754324197769165,
+      "learning_rate": 7.283805878477085e-07,
+      "loss": 0.043,
+      "reward": 2.673539638519287,
+      "reward_std": 0.22582539916038513,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.20146024227142334,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1241
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 248.4,
+      "grad_norm": 0.7383894920349121,
+      "kl": 0.5986624360084534,
+      "learning_rate": 7.245365609417865e-07,
+      "loss": 0.0239,
+      "reward": 1.4299789667129517,
+      "reward_std": 3.6268310546875,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.8200210332870483,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1242
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 248.6,
+      "grad_norm": 0.5437278747558594,
+      "kl": 1.79402494430542,
+      "learning_rate": 7.207009846120718e-07,
+      "loss": 0.0718,
+      "reward": 3.387363910675049,
+      "reward_std": 0.2714118957519531,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.38736385107040405,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1243
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 248.8,
+      "grad_norm": 0.49010857939720154,
+      "kl": 0.6853165626525879,
+      "learning_rate": 7.168738771145465e-07,
+      "loss": 0.0274,
+      "reward": 2.825423240661621,
+      "reward_std": 0.32115474343299866,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.07542331516742706,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1244
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 249.0,
+      "grad_norm": 0.5062938928604126,
+      "kl": 0.8989598155021667,
+      "learning_rate": 7.130552566648847e-07,
+      "loss": 0.036,
+      "reward": 1.902856469154358,
+      "reward_std": 3.6046433448791504,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.6666666865348816,
+      "rewards/wrapped_driving_reward": -0.3888101279735565,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1245
+    },
+    {
+      "completion_length": 454.0,
+      "epoch": 249.2,
+      "grad_norm": 1.461778998374939,
+      "kl": 1.3113372325897217,
+      "learning_rate": 7.092451414383644e-07,
+      "loss": 0.0525,
+      "reward": 2.5652785301208496,
+      "reward_std": 0.5391094088554382,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.3097216486930847,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1246
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 249.4,
+      "grad_norm": 0.5166420936584473,
+      "kl": 0.3338228464126587,
+      "learning_rate": 7.054435495697845e-07,
+      "loss": 0.0134,
+      "reward": 1.3405753374099731,
+      "reward_std": 3.227454662322998,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.0344246625900269,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1247
+    },
+    {
+      "completion_length": 494.0,
+      "epoch": 249.6,
+      "grad_norm": 0.4818384647369385,
+      "kl": 1.1026946306228638,
+      "learning_rate": 7.016504991533727e-07,
+      "loss": 0.0441,
+      "reward": 3.1488592624664307,
+      "reward_std": 0.7905665040016174,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.39885926246643066,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1248
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 249.8,
+      "grad_norm": 0.4349231719970703,
+      "kl": 1.362119197845459,
+      "learning_rate": 6.97866008242703e-07,
+      "loss": 0.0545,
+      "reward": 3.150303840637207,
+      "reward_std": 0.7630019187927246,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.4003037214279175,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1249
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 250.0,
+      "grad_norm": 0.49139872193336487,
+      "kl": 1.5571073293685913,
+      "learning_rate": 6.940900948506113e-07,
+      "loss": 0.0623,
+      "reward": 2.3019723892211914,
+      "reward_std": 0.9020410776138306,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.448027640581131,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1250
+    },
+    {
+      "completion_length": 412.0,
+      "epoch": 250.2,
+      "grad_norm": 0.7658523917198181,
+      "kl": 1.0874384641647339,
+      "learning_rate": 6.903227769491053e-07,
+      "loss": 0.0435,
+      "reward": 3.695434808731079,
+      "reward_std": 0.23894542455673218,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8204347491264343,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1251
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 250.4,
+      "grad_norm": 0.4545027017593384,
+      "kl": 0.8359220623970032,
+      "learning_rate": 6.865640724692815e-07,
+      "loss": 0.0334,
+      "reward": -0.532904863357544,
+      "reward_std": 2.8554065227508545,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -2.907904863357544,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1252
+    },
+    {
+      "completion_length": 423.0,
+      "epoch": 250.6,
+      "grad_norm": 0.8032727241516113,
+      "kl": 0.5447720885276794,
+      "learning_rate": 6.828139993012401e-07,
+      "loss": 0.0218,
+      "reward": 2.4768409729003906,
+      "reward_std": 2.318223237991333,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.5231590270996094,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1253
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 250.8,
+      "grad_norm": 0.6092551350593567,
+      "kl": 0.524155855178833,
+      "learning_rate": 6.790725752939998e-07,
+      "loss": 0.021,
+      "reward": 0.6966722011566162,
+      "reward_std": 2.001295566558838,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -2.303328037261963,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1254
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 251.0,
+      "grad_norm": 0.7360250353813171,
+      "kl": 0.8517738580703735,
+      "learning_rate": 6.753398182554116e-07,
+      "loss": 0.0341,
+      "reward": 2.6352975368499756,
+      "reward_std": 0.2322099655866623,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9772727489471436,
+      "rewards/wrapped_driving_reward": -0.21697509288787842,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1255
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 251.2,
+      "grad_norm": 0.4685967266559601,
+      "kl": 1.0230979919433594,
+      "learning_rate": 6.716157459520739e-07,
+      "loss": 0.0409,
+      "reward": 2.8990492820739746,
+      "reward_std": 0.11218193173408508,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.10095062106847763,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1256
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 251.4,
+      "grad_norm": 0.6057398319244385,
+      "kl": 0.4199396073818207,
+      "learning_rate": 6.679003761092518e-07,
+      "loss": 0.0168,
+      "reward": 1.9826233386993408,
+      "reward_std": 3.3305821418762207,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7272727489471436,
+      "rewards/wrapped_driving_reward": -0.3696495294570923,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1257
+    },
+    {
+      "completion_length": 425.0,
+      "epoch": 251.6,
+      "grad_norm": 0.5870936512947083,
+      "kl": 0.9773305058479309,
+      "learning_rate": 6.641937264107868e-07,
+      "loss": 0.0391,
+      "reward": 3.108414888381958,
+      "reward_std": 0.47746965289115906,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.23341494798660278,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1258
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 251.8,
+      "grad_norm": 0.6660659909248352,
+      "kl": 0.9324990510940552,
+      "learning_rate": 6.604958144990168e-07,
+      "loss": 0.0373,
+      "reward": 1.5900976657867432,
+      "reward_std": 3.4181737899780273,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.6875,
+      "rewards/wrapped_driving_reward": -0.47240233421325684,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1259
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 252.0,
+      "grad_norm": 0.47051286697387695,
+      "kl": 0.5561221241950989,
+      "learning_rate": 6.568066579746901e-07,
+      "loss": 0.0222,
+      "reward": 2.5650317668914795,
+      "reward_std": 0.5829521417617798,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.18496820330619812,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1260
+    },
+    {
+      "completion_length": 496.0,
+      "epoch": 252.2,
+      "grad_norm": 0.4800530970096588,
+      "kl": 0.8208617568016052,
+      "learning_rate": 6.53126274396885e-07,
+      "loss": 0.0328,
+      "reward": 2.8998804092407227,
+      "reward_std": 0.7297057509422302,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.39988023042678833,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 1261
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 252.4,
+      "grad_norm": 0.5117180347442627,
+      "kl": 0.3938012421131134,
+      "learning_rate": 6.494546812829206e-07,
+      "loss": 0.0158,
+      "reward": 1.1696369647979736,
+      "reward_std": 3.114511251449585,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7083333134651184,
+      "rewards/wrapped_driving_reward": -1.1636962890625,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1262
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 252.6,
+      "grad_norm": 0.4717327058315277,
+      "kl": 0.5109540224075317,
+      "learning_rate": 6.457918961082773e-07,
+      "loss": 0.0204,
+      "reward": 1.1229496002197266,
+      "reward_std": 3.48052716255188,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.0020503997802734,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1263
+    },
+    {
+      "completion_length": 391.0,
+      "epoch": 252.8,
+      "grad_norm": 0.5878667235374451,
+      "kl": 0.9013665914535522,
+      "learning_rate": 6.421379363065142e-07,
+      "loss": 0.0361,
+      "reward": 2.121593475341797,
+      "reward_std": 3.414417028427124,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.3784064054489136,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1264
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 253.0,
+      "grad_norm": 0.4330393671989441,
+      "kl": 1.4797282218933105,
+      "learning_rate": 6.384928192691844e-07,
+      "loss": 0.0592,
+      "reward": 2.850269317626953,
+      "reward_std": 0.3921739459037781,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.024730607867240906,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1265
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 253.2,
+      "grad_norm": 0.4989379942417145,
+      "kl": 0.6565256714820862,
+      "learning_rate": 6.348565623457515e-07,
+      "loss": 0.0263,
+      "reward": 2.6121697425842285,
+      "reward_std": 1.2458221912384033,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.012830067425966263,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1266
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 253.4,
+      "grad_norm": 0.4530358910560608,
+      "kl": 0.8784995675086975,
+      "learning_rate": 6.312291828435077e-07,
+      "loss": 0.0351,
+      "reward": 1.2317750453948975,
+      "reward_std": 3.170640468597412,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.65625,
+      "rewards/wrapped_driving_reward": -1.0494749546051025,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1267
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 253.6,
+      "grad_norm": 0.5715232491493225,
+      "kl": 0.9142281413078308,
+      "learning_rate": 6.276106980274946e-07,
+      "loss": 0.0366,
+      "reward": 3.0081124305725098,
+      "reward_std": 0.7514611482620239,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.508112370967865,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 1268
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 253.8,
+      "grad_norm": 0.4626677930355072,
+      "kl": 1.007632851600647,
+      "learning_rate": 6.240011251204153e-07,
+      "loss": 0.0403,
+      "reward": 2.9230029582977295,
+      "reward_std": 0.4594981372356415,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.07699697464704514,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1269
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 254.0,
+      "grad_norm": 0.5885129570960999,
+      "kl": 1.0198560953140259,
+      "learning_rate": 6.204004813025569e-07,
+      "loss": 0.0408,
+      "reward": 1.5203245878219604,
+      "reward_std": 3.3938090801239014,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.6785714626312256,
+      "rewards/wrapped_driving_reward": -0.6582468152046204,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1270
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 254.2,
+      "grad_norm": 0.552034318447113,
+      "kl": 0.669239342212677,
+      "learning_rate": 6.168087837117056e-07,
+      "loss": 0.0268,
+      "reward": -1.3547371625900269,
+      "reward_std": 1.5182576179504395,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -3.6047372817993164,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1271
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 254.4,
+      "grad_norm": 0.5303807854652405,
+      "kl": 0.781219482421875,
+      "learning_rate": 6.1322604944307e-07,
+      "loss": 0.0312,
+      "reward": 1.2259762287139893,
+      "reward_std": 2.148287057876587,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.925000011920929,
+      "rewards/wrapped_driving_reward": -1.6990238428115845,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1272
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 254.6,
+      "grad_norm": 0.47947052121162415,
+      "kl": 0.8710166811943054,
+      "learning_rate": 6.096522955491932e-07,
+      "loss": 0.0348,
+      "reward": 1.2245700359344482,
+      "reward_std": 3.149726629257202,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7250000238418579,
+      "rewards/wrapped_driving_reward": -1.1254298686981201,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1273
+    },
+    {
+      "completion_length": 422.0,
+      "epoch": 254.8,
+      "grad_norm": 0.5638879537582397,
+      "kl": 0.56900554895401,
+      "learning_rate": 6.060875390398757e-07,
+      "loss": 0.0228,
+      "reward": 3.7209606170654297,
+      "reward_std": 0.13869114220142365,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.7209606766700745,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1274
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 255.0,
+      "grad_norm": 0.4698065519332886,
+      "kl": 0.5322856307029724,
+      "learning_rate": 6.025317968820954e-07,
+      "loss": 0.0213,
+      "reward": 2.407771110534668,
+      "reward_std": 2.281691074371338,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9821428656578064,
+      "rewards/wrapped_driving_reward": -0.5743716359138489,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1275
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 255.2,
+      "grad_norm": 0.5505501627922058,
+      "kl": 0.533266007900238,
+      "learning_rate": 5.989850859999227e-07,
+      "loss": 0.0213,
+      "reward": 3.7051382064819336,
+      "reward_std": 0.25479966402053833,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8301382064819336,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1276
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 255.4,
+      "grad_norm": 0.5544794201850891,
+      "kl": 0.6079770922660828,
+      "learning_rate": 5.954474232744439e-07,
+      "loss": 0.0243,
+      "reward": 1.6528925895690918,
+      "reward_std": 3.4379289150238037,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.5971074104309082,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1277
+    },
+    {
+      "completion_length": 440.0,
+      "epoch": 255.6,
+      "grad_norm": 0.5421487092971802,
+      "kl": 0.3778497576713562,
+      "learning_rate": 5.919188255436778e-07,
+      "loss": 0.0151,
+      "reward": 2.499520778656006,
+      "reward_std": 0.5776497721672058,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.2504793405532837,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1278
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 255.8,
+      "grad_norm": 0.4385450482368469,
+      "kl": 1.4436616897583008,
+      "learning_rate": 5.883993096024993e-07,
+      "loss": 0.0577,
+      "reward": 2.202683925628662,
+      "reward_std": 0.8889684081077576,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.96875,
+      "rewards/wrapped_driving_reward": -0.6410658955574036,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1279
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 256.0,
+      "grad_norm": 0.508263349533081,
+      "kl": 0.8775427937507629,
+      "learning_rate": 5.848888922025553e-07,
+      "loss": 0.0351,
+      "reward": 3.2672133445739746,
+      "reward_std": 0.18384453654289246,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.39221322536468506,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1280
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 256.2,
+      "grad_norm": 0.5863519310951233,
+      "kl": 1.4571540355682373,
+      "learning_rate": 5.813875900521868e-07,
+      "loss": 0.0583,
+      "reward": 2.7691006660461426,
+      "reward_std": 0.30246564745903015,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.10589944571256638,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1281
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 256.4,
+      "grad_norm": 0.47644615173339844,
+      "kl": 1.624181866645813,
+      "learning_rate": 5.778954198163514e-07,
+      "loss": 0.065,
+      "reward": 3.382758140563965,
+      "reward_std": 0.3809010088443756,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.38275817036628723,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1282
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 256.6,
+      "grad_norm": 0.5348938703536987,
+      "kl": 0.31698116660118103,
+      "learning_rate": 5.7441239811654e-07,
+      "loss": 0.0127,
+      "reward": 0.8874088525772095,
+      "reward_std": 2.7571182250976562,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.8625911474227905,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1283
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 256.8,
+      "grad_norm": 0.6255142688751221,
+      "kl": 0.4159882962703705,
+      "learning_rate": 5.709385415307006e-07,
+      "loss": 0.0166,
+      "reward": 3.3938422203063965,
+      "reward_std": 0.5522288680076599,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.7688423991203308,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1284
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 257.0,
+      "grad_norm": 0.5825262069702148,
+      "kl": 0.7926734089851379,
+      "learning_rate": 5.674738665931575e-07,
+      "loss": 0.0317,
+      "reward": 3.180075168609619,
+      "reward_std": 0.2703258991241455,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.18007516860961914,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1285
+    },
+    {
+      "completion_length": 463.0,
+      "epoch": 257.2,
+      "grad_norm": 0.5172073245048523,
+      "kl": 0.8048291206359863,
+      "learning_rate": 5.640183897945362e-07,
+      "loss": 0.0322,
+      "reward": 1.1660841703414917,
+      "reward_std": 2.080986976623535,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.5839158296585083,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1286
+    },
+    {
+      "completion_length": 381.0,
+      "epoch": 257.4,
+      "grad_norm": 0.5643807053565979,
+      "kl": 0.5665979981422424,
+      "learning_rate": 5.605721275816797e-07,
+      "loss": 0.0227,
+      "reward": 3.7975504398345947,
+      "reward_std": 0.038680512458086014,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.7975503206253052,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1287
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 257.6,
+      "grad_norm": 0.51383376121521,
+      "kl": 0.6882296204566956,
+      "learning_rate": 5.571350963575728e-07,
+      "loss": 0.0275,
+      "reward": 3.00128436088562,
+      "reward_std": 0.42292237281799316,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9615384340286255,
+      "rewards/wrapped_driving_reward": 0.28974586725234985,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1288
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 257.8,
+      "grad_norm": 0.4541057348251343,
+      "kl": 0.49609461426734924,
+      "learning_rate": 5.537073124812667e-07,
+      "loss": 0.0198,
+      "reward": 1.4493234157562256,
+      "reward_std": 1.5718995332717896,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9545454382896423,
+      "rewards/wrapped_driving_reward": -1.130221962928772,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1289
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 258.0,
+      "grad_norm": 0.5707848072052002,
+      "kl": 1.1001707315444946,
+      "learning_rate": 5.50288792267796e-07,
+      "loss": 0.044,
+      "reward": 3.174471378326416,
+      "reward_std": 0.6302404999732971,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.6744711399078369,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 1290
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 258.2,
+      "grad_norm": 0.5454885363578796,
+      "kl": 0.5242068767547607,
+      "learning_rate": 5.468795519881043e-07,
+      "loss": 0.021,
+      "reward": 0.823435366153717,
+      "reward_std": 3.3484692573547363,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.3015645742416382,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1291
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 258.4,
+      "grad_norm": 3.361776351928711,
+      "kl": 1.0555751323699951,
+      "learning_rate": 5.434796078689653e-07,
+      "loss": 0.0422,
+      "reward": 1.6648452281951904,
+      "reward_std": 3.129169464111328,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.625,
+      "rewards/wrapped_driving_reward": -0.5851547122001648,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1292
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 258.6,
+      "grad_norm": 0.4585336446762085,
+      "kl": 0.6999787092208862,
+      "learning_rate": 5.400889760929088e-07,
+      "loss": 0.028,
+      "reward": 3.692640781402588,
+      "reward_std": 0.2535787522792816,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9750000238418579,
+      "rewards/wrapped_driving_reward": 0.7176405787467957,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1293
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 258.8,
+      "grad_norm": 0.5164147615432739,
+      "kl": 1.2436740398406982,
+      "learning_rate": 5.367076727981383e-07,
+      "loss": 0.0497,
+      "reward": 2.5147736072540283,
+      "reward_std": 0.5300504565238953,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.23522637784481049,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1294
+    },
+    {
+      "completion_length": 437.0,
+      "epoch": 259.0,
+      "grad_norm": 0.5326743721961975,
+      "kl": 0.6541189551353455,
+      "learning_rate": 5.333357140784576e-07,
+      "loss": 0.0262,
+      "reward": 3.1108293533325195,
+      "reward_std": 0.26085424423217773,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.23582951724529266,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1295
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 259.2,
+      "grad_norm": 0.4311022162437439,
+      "kl": 1.186979055404663,
+      "learning_rate": 5.299731159831953e-07,
+      "loss": 0.0475,
+      "reward": 3.1702427864074707,
+      "reward_std": 0.3770046532154083,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.17024272680282593,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1296
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 259.4,
+      "grad_norm": 0.47780653834342957,
+      "kl": 0.8817828893661499,
+      "learning_rate": 5.266198945171253e-07,
+      "loss": 0.0353,
+      "reward": 2.6567845344543457,
+      "reward_std": 0.3220916986465454,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.96875,
+      "rewards/wrapped_driving_reward": -0.18696540594100952,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1297
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 259.6,
+      "grad_norm": 3.290817975997925,
+      "kl": 1.8572345972061157,
+      "learning_rate": 5.232760656403923e-07,
+      "loss": 0.0743,
+      "reward": 3.702077865600586,
+      "reward_std": 0.26143285632133484,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8270776867866516,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1298
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 259.8,
+      "grad_norm": 0.5228665471076965,
+      "kl": 0.5127549171447754,
+      "learning_rate": 5.199416452684347e-07,
+      "loss": 0.0205,
+      "reward": -0.543779730796814,
+      "reward_std": 2.8356096744537354,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -2.9187798500061035,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1299
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 260.0,
+      "grad_norm": 0.5405871272087097,
+      "kl": 0.9794502854347229,
+      "learning_rate": 5.166166492719124e-07,
+      "loss": 0.0392,
+      "reward": 3.104926824569702,
+      "reward_std": 0.7653807997703552,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.875,
+      "rewards/wrapped_driving_reward": 0.3549269437789917,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1300
+    },
+    {
+      "completion_length": 461.0,
+      "epoch": 260.2,
+      "grad_norm": 0.510883092880249,
+      "kl": 0.6124632954597473,
+      "learning_rate": 5.133010934766264e-07,
+      "loss": 0.0245,
+      "reward": 1.7851228713989258,
+      "reward_std": 1.8598078489303589,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9444444179534912,
+      "rewards/wrapped_driving_reward": -1.1593215465545654,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1301
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 260.4,
+      "grad_norm": 0.4719240665435791,
+      "kl": 1.3469144105911255,
+      "learning_rate": 5.099949936634452e-07,
+      "loss": 0.0539,
+      "reward": 1.3794589042663574,
+      "reward_std": 2.110288143157959,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.949999988079071,
+      "rewards/wrapped_driving_reward": -1.3205411434173584,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1302
+    },
+    {
+      "completion_length": 415.0,
+      "epoch": 260.6,
+      "grad_norm": 0.6128676533699036,
+      "kl": 0.802237331867218,
+      "learning_rate": 5.066983655682325e-07,
+      "loss": 0.0321,
+      "reward": 3.548816680908203,
+      "reward_std": 0.30427923798561096,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.6738166809082031,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1303
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 260.8,
+      "grad_norm": 0.47834450006484985,
+      "kl": 0.7036153078079224,
+      "learning_rate": 5.034112248817685e-07,
+      "loss": 0.0281,
+      "reward": 1.7580673694610596,
+      "reward_std": 1.880155324935913,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.11693274974823,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1304
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 261.0,
+      "grad_norm": 0.5236107110977173,
+      "kl": 0.5109760761260986,
+      "learning_rate": 5.001335872496759e-07,
+      "loss": 0.0204,
+      "reward": 1.7466497421264648,
+      "reward_std": 3.528158187866211,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.3783501982688904,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1305
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 261.2,
+      "grad_norm": 0.6448484063148499,
+      "kl": 0.7483813166618347,
+      "learning_rate": 4.968654682723487e-07,
+      "loss": 0.0299,
+      "reward": 1.1037460565567017,
+      "reward_std": 1.7360997200012207,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.7712539434432983,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1306
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 261.4,
+      "grad_norm": 0.5170474052429199,
+      "kl": 0.44401609897613525,
+      "learning_rate": 4.936068835048733e-07,
+      "loss": 0.0178,
+      "reward": 1.1237719058990479,
+      "reward_std": 3.0878007411956787,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.5012280941009521,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 1307
+    },
+    {
+      "completion_length": 452.0,
+      "epoch": 261.6,
+      "grad_norm": 1.000315546989441,
+      "kl": 0.9111926555633545,
+      "learning_rate": 4.903578484569568e-07,
+      "loss": 0.0364,
+      "reward": 3.246818780899048,
+      "reward_std": 0.7480571866035461,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": 0.4968188405036926,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1308
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 261.8,
+      "grad_norm": 0.45820820331573486,
+      "kl": 0.5465204119682312,
+      "learning_rate": 4.871183785928546e-07,
+      "loss": 0.0219,
+      "reward": 0.5374484062194824,
+      "reward_std": 1.777597427368164,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -2.2125515937805176,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1309
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 262.0,
+      "grad_norm": 0.4779776930809021,
+      "kl": 1.0618098974227905,
+      "learning_rate": 4.838884893312934e-07,
+      "loss": 0.0425,
+      "reward": 1.5686064958572388,
+      "reward_std": 3.717726707458496,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.6813935041427612,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1310
+    },
+    {
+      "completion_length": 430.0,
+      "epoch": 262.2,
+      "grad_norm": 0.6215850114822388,
+      "kl": 1.6949265003204346,
+      "learning_rate": 4.806681960454012e-07,
+      "loss": 0.0678,
+      "reward": 2.0167996883392334,
+      "reward_std": 2.0115416049957275,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.8582002520561218,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1311
+    },
+    {
+      "completion_length": 491.0,
+      "epoch": 262.4,
+      "grad_norm": 0.4644632041454315,
+      "kl": 0.913459062576294,
+      "learning_rate": 4.774575140626317e-07,
+      "loss": 0.0365,
+      "reward": 2.6541190147399902,
+      "reward_std": 0.5420801043510437,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.22088098526000977,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1312
+    },
+    {
+      "completion_length": 431.0,
+      "epoch": 262.6,
+      "grad_norm": 0.4364088773727417,
+      "kl": 1.2697290182113647,
+      "learning_rate": 4.7425645866469113e-07,
+      "loss": 0.0508,
+      "reward": 2.789670944213867,
+      "reward_std": 0.5778433680534363,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.21032902598381042,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1313
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 262.8,
+      "grad_norm": 0.8658943772315979,
+      "kl": 0.7999235391616821,
+      "learning_rate": 4.710650450874693e-07,
+      "loss": 0.032,
+      "reward": 3.515301465988159,
+      "reward_std": 0.4819580018520355,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.7653014659881592,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1314
+    },
+    {
+      "completion_length": 490.0,
+      "epoch": 263.0,
+      "grad_norm": 0.49543485045433044,
+      "kl": 0.8240687251091003,
+      "learning_rate": 4.678832885209622e-07,
+      "loss": 0.033,
+      "reward": 2.9809787273406982,
+      "reward_std": 0.4203469753265381,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9642857313156128,
+      "rewards/wrapped_driving_reward": 0.2666930854320526,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1315
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 263.2,
+      "grad_norm": 0.49222421646118164,
+      "kl": 1.3794502019882202,
+      "learning_rate": 4.6471120410920225e-07,
+      "loss": 0.0552,
+      "reward": 2.681150197982788,
+      "reward_std": 0.612041711807251,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.06884972751140594,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1316
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 263.4,
+      "grad_norm": 0.7817544341087341,
+      "kl": 0.6659716367721558,
+      "learning_rate": 4.6154880695018585e-07,
+      "loss": 0.0266,
+      "reward": 3.1257710456848145,
+      "reward_std": 0.5564408898353577,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8863636255264282,
+      "rewards/wrapped_driving_reward": 0.2394072562456131,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1317
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 263.6,
+      "grad_norm": 0.44470110535621643,
+      "kl": 0.5849674940109253,
+      "learning_rate": 4.5839611209580277e-07,
+      "loss": 0.0234,
+      "reward": 2.9830784797668457,
+      "reward_std": 0.7209075093269348,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.016921691596508026,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1318
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 263.8,
+      "grad_norm": 0.5052596926689148,
+      "kl": 1.1991493701934814,
+      "learning_rate": 4.5525313455176165e-07,
+      "loss": 0.048,
+      "reward": 1.954295039176941,
+      "reward_std": 3.636528730392456,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.71875,
+      "rewards/wrapped_driving_reward": -0.38945502042770386,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1319
+    },
+    {
+      "completion_length": 483.0,
+      "epoch": 264.0,
+      "grad_norm": 0.7518298029899597,
+      "kl": 1.6993350982666016,
+      "learning_rate": 4.5211988927752026e-07,
+      "loss": 0.068,
+      "reward": 2.980362892150879,
+      "reward_std": 0.8009577393531799,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.35536280274391174,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1320
+    },
+    {
+      "completion_length": 471.0,
+      "epoch": 264.2,
+      "grad_norm": 0.5145017504692078,
+      "kl": 1.43527352809906,
+      "learning_rate": 4.4899639118621606e-07,
+      "loss": 0.0574,
+      "reward": 2.5778322219848633,
+      "reward_std": 0.5218800902366638,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.17216773331165314,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1321
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 264.4,
+      "grad_norm": 0.511737585067749,
+      "kl": 0.7654083967208862,
+      "learning_rate": 4.458826551445911e-07,
+      "loss": 0.0306,
+      "reward": 2.8301141262054443,
+      "reward_std": 0.7655457258224487,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.044885873794555664,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1322
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 264.6,
+      "grad_norm": 0.7087801098823547,
+      "kl": 1.2927275896072388,
+      "learning_rate": 4.427786959729247e-07,
+      "loss": 0.0517,
+      "reward": 1.9969390630722046,
+      "reward_std": 3.6646361351013184,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.3780609369277954,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1323
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 264.8,
+      "grad_norm": 0.5631241798400879,
+      "kl": 0.6530361771583557,
+      "learning_rate": 4.396845284449608e-07,
+      "loss": 0.0261,
+      "reward": 1.7299089431762695,
+      "reward_std": 3.50333571434021,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.64509117603302,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1324
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 265.0,
+      "grad_norm": 0.6352676749229431,
+      "kl": 1.028291940689087,
+      "learning_rate": 4.366001672878406e-07,
+      "loss": 0.0411,
+      "reward": 1.79507315158844,
+      "reward_std": 1.8661749362945557,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.20492684841156,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1325
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 265.2,
+      "grad_norm": 0.5242295265197754,
+      "kl": 0.30665263533592224,
+      "learning_rate": 4.335256271820287e-07,
+      "loss": 0.0123,
+      "reward": 3.2212300300598145,
+      "reward_std": 0.5791422128677368,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.875,
+      "rewards/wrapped_driving_reward": 0.5962298512458801,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1326
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 265.4,
+      "grad_norm": 0.5242313742637634,
+      "kl": 0.7018243074417114,
+      "learning_rate": 4.3046092276124467e-07,
+      "loss": 0.0281,
+      "reward": 0.01702582836151123,
+      "reward_std": 2.0340516567230225,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -2.9829742908477783,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1327
+    },
+    {
+      "completion_length": 486.0,
+      "epoch": 265.6,
+      "grad_norm": 0.4481220543384552,
+      "kl": 0.8071734309196472,
+      "learning_rate": 4.27406068612396e-07,
+      "loss": 0.0323,
+      "reward": 3.0400636196136475,
+      "reward_std": 0.4359326958656311,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9375,
+      "rewards/wrapped_driving_reward": 0.10256364196538925,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1328
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 265.8,
+      "grad_norm": 0.5211302638053894,
+      "kl": 0.6702889204025269,
+      "learning_rate": 4.2436107927550373e-07,
+      "loss": 0.0268,
+      "reward": 1.9131126403808594,
+      "reward_std": 3.612079620361328,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.4618873596191406,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1329
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 266.0,
+      "grad_norm": 0.5514548420906067,
+      "kl": 1.2058435678482056,
+      "learning_rate": 4.2132596924363666e-07,
+      "loss": 0.0482,
+      "reward": 1.6061519384384155,
+      "reward_std": 3.07539439201355,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.7688480615615845,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1330
+    },
+    {
+      "completion_length": 475.0,
+      "epoch": 266.2,
+      "grad_norm": 0.7321079969406128,
+      "kl": 0.779521107673645,
+      "learning_rate": 4.1830075296284354e-07,
+      "loss": 0.0312,
+      "reward": 0.4196197986602783,
+      "reward_std": 3.949946880340576,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.5803802013397217,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1331
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 266.4,
+      "grad_norm": 6.919052600860596,
+      "kl": 1.576327919960022,
+      "learning_rate": 4.152854448320798e-07,
+      "loss": 0.0631,
+      "reward": 2.3543877601623535,
+      "reward_std": 0.587752103805542,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.39561226963996887,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1332
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 266.6,
+      "grad_norm": 0.4669599235057831,
+      "kl": 1.43578040599823,
+      "learning_rate": 4.122800592031426e-07,
+      "loss": 0.0574,
+      "reward": 2.4777488708496094,
+      "reward_std": 2.337543249130249,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.5222511291503906,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1333
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 266.8,
+      "grad_norm": 0.4864882826805115,
+      "kl": 1.3176442384719849,
+      "learning_rate": 4.0928461038060114e-07,
+      "loss": 0.0527,
+      "reward": 2.989267110824585,
+      "reward_std": 0.4231380224227905,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9750000238418579,
+      "rewards/wrapped_driving_reward": 0.13926707208156586,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1334
+    },
+    {
+      "completion_length": 459.0,
+      "epoch": 267.0,
+      "grad_norm": 0.8441217541694641,
+      "kl": 1.4224587678909302,
+      "learning_rate": 4.0629911262173053e-07,
+      "loss": 0.0569,
+      "reward": 2.697777271270752,
+      "reward_std": 0.2736762464046478,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.987500011920929,
+      "rewards/wrapped_driving_reward": -0.2897225320339203,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1335
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 267.2,
+      "grad_norm": 0.5214635133743286,
+      "kl": 0.8030060529708862,
+      "learning_rate": 4.033235801364402e-07,
+      "loss": 0.0321,
+      "reward": 1.5438249111175537,
+      "reward_std": 3.388503074645996,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.7061751484870911,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1336
+    },
+    {
+      "completion_length": 350.0,
+      "epoch": 267.4,
+      "grad_norm": 0.639860987663269,
+      "kl": 0.5820645689964294,
+      "learning_rate": 4.003580270872093e-07,
+      "loss": 0.0233,
+      "reward": 0.3889540433883667,
+      "reward_std": 3.2138869762420654,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.9860459566116333,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1337
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 267.6,
+      "grad_norm": 0.4588041305541992,
+      "kl": 0.6945257186889648,
+      "learning_rate": 3.97402467589019e-07,
+      "loss": 0.0278,
+      "reward": 2.6269006729125977,
+      "reward_std": 1.047061562538147,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9375,
+      "rewards/wrapped_driving_reward": -0.06059914827346802,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1338
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 267.8,
+      "grad_norm": 0.47228652238845825,
+      "kl": 0.7689561247825623,
+      "learning_rate": 3.94456915709284e-07,
+      "loss": 0.0308,
+      "reward": 1.9588474035263062,
+      "reward_std": 3.639615297317505,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7250000238418579,
+      "rewards/wrapped_driving_reward": -0.39115265011787415,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1339
+    },
+    {
+      "completion_length": 481.0,
+      "epoch": 268.0,
+      "grad_norm": 0.5430285334587097,
+      "kl": 0.3783856928348541,
+      "learning_rate": 3.915213854677863e-07,
+      "loss": 0.0151,
+      "reward": 3.194617986679077,
+      "reward_std": 0.36717328429222107,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.875,
+      "rewards/wrapped_driving_reward": 0.3196178674697876,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1340
+    },
+    {
+      "completion_length": 479.0,
+      "epoch": 268.2,
+      "grad_norm": 0.4091739058494568,
+      "kl": 1.440745234489441,
+      "learning_rate": 3.8859589083660803e-07,
+      "loss": 0.0576,
+      "reward": 3.829369068145752,
+      "reward_std": 0.002827721880748868,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8293691277503967,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1341
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 268.4,
+      "grad_norm": 0.44796013832092285,
+      "kl": 1.121548056602478,
+      "learning_rate": 3.85680445740067e-07,
+      "loss": 0.0449,
+      "reward": 1.6584455966949463,
+      "reward_std": 3.439409017562866,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.7165544033050537,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1342
+    },
+    {
+      "completion_length": 450.0,
+      "epoch": 268.6,
+      "grad_norm": 0.513882040977478,
+      "kl": 1.2204365730285645,
+      "learning_rate": 3.8277506405464763e-07,
+      "loss": 0.0488,
+      "reward": 2.934741973876953,
+      "reward_std": 0.687610924243927,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9772727489471436,
+      "rewards/wrapped_driving_reward": -0.04253065586090088,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1343
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 268.8,
+      "grad_norm": 0.5922746062278748,
+      "kl": 1.8003734350204468,
+      "learning_rate": 3.798797596089351e-07,
+      "loss": 0.072,
+      "reward": 3.182130813598633,
+      "reward_std": 0.3506509065628052,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.1821308135986328,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1344
+    },
+    {
+      "completion_length": 456.0,
+      "epoch": 269.0,
+      "grad_norm": 0.549982488155365,
+      "kl": 0.9568000435829163,
+      "learning_rate": 3.7699454618355306e-07,
+      "loss": 0.0383,
+      "reward": 2.6680068969726562,
+      "reward_std": 0.24568282067775726,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.987500011920929,
+      "rewards/wrapped_driving_reward": -0.1944931298494339,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1345
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 269.2,
+      "grad_norm": 0.4835640788078308,
+      "kl": 0.6245141625404358,
+      "learning_rate": 3.741194375110932e-07,
+      "loss": 0.025,
+      "reward": 1.754793405532837,
+      "reward_std": 2.457003593444824,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8928571343421936,
+      "rewards/wrapped_driving_reward": -0.6380638480186462,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 1346
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 269.4,
+      "grad_norm": 0.5082364678382874,
+      "kl": 0.7363067269325256,
+      "learning_rate": 3.712544472760532e-07,
+      "loss": 0.0295,
+      "reward": 3.556278705596924,
+      "reward_std": 0.328625351190567,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.6812785863876343,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1347
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 269.6,
+      "grad_norm": 0.7735382318496704,
+      "kl": 0.8911629915237427,
+      "learning_rate": 3.683995891147696e-07,
+      "loss": 0.0356,
+      "reward": 1.21405029296875,
+      "reward_std": 3.4829022884368896,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.7859497666358948,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 1348
+    },
+    {
+      "completion_length": 430.0,
+      "epoch": 269.8,
+      "grad_norm": 0.5301557183265686,
+      "kl": 1.0712778568267822,
+      "learning_rate": 3.6555487661535604e-07,
+      "loss": 0.0429,
+      "reward": 2.8502888679504395,
+      "reward_std": 0.5803841352462769,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.14971117675304413,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1349
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 270.0,
+      "grad_norm": 0.727816641330719,
+      "kl": 0.7259580492973328,
+      "learning_rate": 3.627203233176341e-07,
+      "loss": 0.029,
+      "reward": 2.5493853092193604,
+      "reward_std": 0.3004813492298126,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.20061470568180084,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1350
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 270.2,
+      "grad_norm": 0.6522797346115112,
+      "kl": 1.103890299797058,
+      "learning_rate": 3.598959427130716e-07,
+      "loss": 0.0442,
+      "reward": 3.2470264434814453,
+      "reward_std": 0.23121625185012817,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9236111044883728,
+      "rewards/wrapped_driving_reward": 0.3234153389930725,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1351
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 270.4,
+      "grad_norm": 0.4576016366481781,
+      "kl": 1.3178799152374268,
+      "learning_rate": 3.5708174824471947e-07,
+      "loss": 0.0527,
+      "reward": 1.1758347749710083,
+      "reward_std": 2.58150315284729,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9722222089767456,
+      "rewards/wrapped_driving_reward": -1.7963873147964478,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1352
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 270.6,
+      "grad_norm": 0.5248624086380005,
+      "kl": 0.3703576922416687,
+      "learning_rate": 3.542777533071443e-07,
+      "loss": 0.0148,
+      "reward": 0.05720162391662598,
+      "reward_std": 4.400914192199707,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.567798376083374,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1353
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 270.8,
+      "grad_norm": 0.5428052544593811,
+      "kl": 0.5293287038803101,
+      "learning_rate": 3.514839712463683e-07,
+      "loss": 0.0212,
+      "reward": 1.543633222579956,
+      "reward_std": 3.3753809928894043,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7250000238418579,
+      "rewards/wrapped_driving_reward": -0.8063667416572571,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1354
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 271.0,
+      "grad_norm": 0.4755152761936188,
+      "kl": 1.3337355852127075,
+      "learning_rate": 3.4870041535980283e-07,
+      "loss": 0.0533,
+      "reward": 2.520254611968994,
+      "reward_std": 0.04152216762304306,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.4797452390193939,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1355
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 271.2,
+      "grad_norm": 0.5265377163887024,
+      "kl": 1.3497384786605835,
+      "learning_rate": 3.4592709889618546e-07,
+      "loss": 0.054,
+      "reward": 2.603036880493164,
+      "reward_std": 0.23215869069099426,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.27196308970451355,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1356
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 271.4,
+      "grad_norm": 0.48459577560424805,
+      "kl": 0.3322940766811371,
+      "learning_rate": 3.4316403505552045e-07,
+      "loss": 0.0133,
+      "reward": 3.1807594299316406,
+      "reward_std": 0.9502668380737305,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.96875,
+      "rewards/wrapped_driving_reward": 0.46200960874557495,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1357
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 271.6,
+      "grad_norm": 0.6433394551277161,
+      "kl": 0.6385685801506042,
+      "learning_rate": 3.4041123698901086e-07,
+      "loss": 0.0255,
+      "reward": 2.894571304321289,
+      "reward_std": 0.49757829308509827,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.14457115530967712,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1358
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 271.8,
+      "grad_norm": 0.6986045241355896,
+      "kl": 1.0710160732269287,
+      "learning_rate": 3.3766871779899875e-07,
+      "loss": 0.0428,
+      "reward": 3.2389633655548096,
+      "reward_std": 0.7566831111907959,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.48896336555480957,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1359
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 272.0,
+      "grad_norm": 0.4622398912906647,
+      "kl": 1.2733514308929443,
+      "learning_rate": 3.3493649053890325e-07,
+      "loss": 0.0509,
+      "reward": 2.77608060836792,
+      "reward_std": 0.20391617715358734,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.09891960769891739,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1360
+    },
+    {
+      "completion_length": 461.0,
+      "epoch": 272.2,
+      "grad_norm": 0.5052734017372131,
+      "kl": 0.9263938069343567,
+      "learning_rate": 3.3221456821315693e-07,
+      "loss": 0.0371,
+      "reward": 3.5572261810302734,
+      "reward_std": 0.5255562663078308,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8072260618209839,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1361
+    },
+    {
+      "completion_length": 470.0,
+      "epoch": 272.4,
+      "grad_norm": 0.6449248790740967,
+      "kl": 1.497145652770996,
+      "learning_rate": 3.295029637771441e-07,
+      "loss": 0.0599,
+      "reward": 2.9099924564361572,
+      "reward_std": 0.4086282253265381,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.15999245643615723,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1362
+    },
+    {
+      "completion_length": 449.0,
+      "epoch": 272.6,
+      "grad_norm": 0.5337421298027039,
+      "kl": 1.26763117313385,
+      "learning_rate": 3.268016901371407e-07,
+      "loss": 0.0507,
+      "reward": 1.5956735610961914,
+      "reward_std": 2.399942636489868,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9722222089767456,
+      "rewards/wrapped_driving_reward": -1.1265487670898438,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1363
+    },
+    {
+      "completion_length": 373.0,
+      "epoch": 272.8,
+      "grad_norm": 0.5712352395057678,
+      "kl": 0.4801625609397888,
+      "learning_rate": 3.241107601502508e-07,
+      "loss": 0.0192,
+      "reward": 2.3176591396331787,
+      "reward_std": 2.211979389190674,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.6823408007621765,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1364
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 273.0,
+      "grad_norm": 0.46319812536239624,
+      "kl": 0.5127870440483093,
+      "learning_rate": 3.214301866243469e-07,
+      "loss": 0.0205,
+      "reward": 2.531925916671753,
+      "reward_std": 2.356149673461914,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9611110687255859,
+      "rewards/wrapped_driving_reward": -0.42918533086776733,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1365
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 273.2,
+      "grad_norm": 0.4961586594581604,
+      "kl": 0.7962093949317932,
+      "learning_rate": 3.187599823180071e-07,
+      "loss": 0.0318,
+      "reward": 2.5397024154663086,
+      "reward_std": 0.34739187359809875,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.33529773354530334,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1366
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 273.4,
+      "grad_norm": 0.495364785194397,
+      "kl": 0.7481328248977661,
+      "learning_rate": 3.161001599404581e-07,
+      "loss": 0.0299,
+      "reward": 2.379721164703369,
+      "reward_std": 0.38598403334617615,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.12027896195650101,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 1367
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 273.6,
+      "grad_norm": 0.6948127150535583,
+      "kl": 0.9188811779022217,
+      "learning_rate": 3.134507321515107e-07,
+      "loss": 0.0368,
+      "reward": 3.2667016983032227,
+      "reward_std": 0.592399001121521,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.5167015194892883,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1368
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 273.8,
+      "grad_norm": 0.6983757615089417,
+      "kl": 1.0416016578674316,
+      "learning_rate": 3.108117115615006e-07,
+      "loss": 0.0417,
+      "reward": 2.1769251823425293,
+      "reward_std": 0.9528898596763611,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9272727370262146,
+      "rewards/wrapped_driving_reward": -0.6253474950790405,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1369
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 274.0,
+      "grad_norm": 0.6631730794906616,
+      "kl": 0.5493196249008179,
+      "learning_rate": 3.081831107312308e-07,
+      "loss": 0.022,
+      "reward": 1.3180338144302368,
+      "reward_std": 3.271329164505005,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.8069661855697632,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1370
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 274.2,
+      "grad_norm": 0.44483792781829834,
+      "kl": 0.22444850206375122,
+      "learning_rate": 3.0556494217190843e-07,
+      "loss": 0.009,
+      "reward": 2.6468000411987305,
+      "reward_std": 0.4043002724647522,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.35319995880126953,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1371
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 274.4,
+      "grad_norm": 0.7638071775436401,
+      "kl": 0.6436581015586853,
+      "learning_rate": 3.0295721834508686e-07,
+      "loss": 0.0257,
+      "reward": 1.1492899656295776,
+      "reward_std": 3.4339349269866943,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.1007100343704224,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1372
+    },
+    {
+      "completion_length": 472.0,
+      "epoch": 274.6,
+      "grad_norm": 0.49714797735214233,
+      "kl": 0.9065066576004028,
+      "learning_rate": 3.0035995166260656e-07,
+      "loss": 0.0363,
+      "reward": 3.0749258995056152,
+      "reward_std": 0.44597530364990234,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9583333134651184,
+      "rewards/wrapped_driving_reward": 0.3665924668312073,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1373
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 274.8,
+      "grad_norm": 0.5556493401527405,
+      "kl": 1.1727126836776733,
+      "learning_rate": 2.9777315448653615e-07,
+      "loss": 0.0469,
+      "reward": 1.4110647439956665,
+      "reward_std": 2.3108038902282715,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.0889352560043335,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 1374
+    },
+    {
+      "completion_length": 486.0,
+      "epoch": 275.0,
+      "grad_norm": 0.45533323287963867,
+      "kl": 1.0475839376449585,
+      "learning_rate": 2.9519683912911267e-07,
+      "loss": 0.0419,
+      "reward": 2.6271228790283203,
+      "reward_std": 0.9351125955581665,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.96875,
+      "rewards/wrapped_driving_reward": -0.21662718057632446,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1375
+    },
+    {
+      "completion_length": 491.0,
+      "epoch": 275.2,
+      "grad_norm": 0.4396159052848816,
+      "kl": 1.2950646877288818,
+      "learning_rate": 2.9263101785268253e-07,
+      "loss": 0.0518,
+      "reward": 3.436497449874878,
+      "reward_std": 0.19118525087833405,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.43649736046791077,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1376
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 275.4,
+      "grad_norm": 0.5278507471084595,
+      "kl": 1.3072184324264526,
+      "learning_rate": 2.90075702869646e-07,
+      "loss": 0.0523,
+      "reward": 3.100782871246338,
+      "reward_std": 0.21688224375247955,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.1007828563451767,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1377
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 275.6,
+      "grad_norm": 0.4898889660835266,
+      "kl": 0.7671000361442566,
+      "learning_rate": 2.875309063423956e-07,
+      "loss": 0.0307,
+      "reward": 3.2323617935180664,
+      "reward_std": 0.7623991966247559,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9791666865348816,
+      "rewards/wrapped_driving_reward": 0.50319504737854,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1378
+    },
+    {
+      "completion_length": 476.0,
+      "epoch": 275.8,
+      "grad_norm": 0.5012227892875671,
+      "kl": 0.898972749710083,
+      "learning_rate": 2.849966403832599e-07,
+      "loss": 0.036,
+      "reward": 3.2697856426239014,
+      "reward_std": 0.6626085042953491,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.96875,
+      "rewards/wrapped_driving_reward": 0.3010355830192566,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1379
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 276.0,
+      "grad_norm": 0.5459585189819336,
+      "kl": 1.2240800857543945,
+      "learning_rate": 2.8247291705444575e-07,
+      "loss": 0.049,
+      "reward": 2.81314754486084,
+      "reward_std": 0.04792924225330353,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.18685244023799896,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1380
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 276.2,
+      "grad_norm": 0.5124369263648987,
+      "kl": 0.6807260513305664,
+      "learning_rate": 2.7995974836798194e-07,
+      "loss": 0.0272,
+      "reward": 1.8669229745864868,
+      "reward_std": 1.9425448179244995,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.1330770254135132,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1381
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 276.4,
+      "grad_norm": 0.4732033908367157,
+      "kl": 0.5008506774902344,
+      "learning_rate": 2.774571462856593e-07,
+      "loss": 0.02,
+      "reward": 3.8286309242248535,
+      "reward_std": 0.02097063697874546,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8286310434341431,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1382
+    },
+    {
+      "completion_length": 450.0,
+      "epoch": 276.6,
+      "grad_norm": 0.47596240043640137,
+      "kl": 1.2041912078857422,
+      "learning_rate": 2.749651227189756e-07,
+      "loss": 0.0482,
+      "reward": 3.006807804107666,
+      "reward_std": 0.42060747742652893,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.006807684898376465,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1383
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 276.8,
+      "grad_norm": 0.5646078586578369,
+      "kl": 1.236280918121338,
+      "learning_rate": 2.7248368952908055e-07,
+      "loss": 0.0495,
+      "reward": 2.1746325492858887,
+      "reward_std": 2.4662575721740723,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.7003673315048218,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1384
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 277.0,
+      "grad_norm": 0.5159252285957336,
+      "kl": 1.3205516338348389,
+      "learning_rate": 2.700128585267148e-07,
+      "loss": 0.0528,
+      "reward": -0.7608491778373718,
+      "reward_std": 2.447402238845825,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -3.1358492374420166,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1385
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 277.2,
+      "grad_norm": 0.8124915361404419,
+      "kl": 0.8083114624023438,
+      "learning_rate": 2.6755264147215796e-07,
+      "loss": 0.0323,
+      "reward": 3.094942569732666,
+      "reward_std": 0.6148591637611389,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.3449426293373108,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1386
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 277.4,
+      "grad_norm": 0.4281749427318573,
+      "kl": 1.3329992294311523,
+      "learning_rate": 2.6510305007516974e-07,
+      "loss": 0.0533,
+      "reward": 2.8872785568237305,
+      "reward_std": 0.4396892189979553,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.11272148042917252,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1387
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 277.6,
+      "grad_norm": 0.4807472825050354,
+      "kl": 0.8092159628868103,
+      "learning_rate": 2.6266409599493753e-07,
+      "loss": 0.0324,
+      "reward": 2.819895029067993,
+      "reward_std": 0.6123683452606201,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.06989501416683197,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1388
+    },
+    {
+      "completion_length": 480.0,
+      "epoch": 277.8,
+      "grad_norm": 0.5461340546607971,
+      "kl": 0.7984523177146912,
+      "learning_rate": 2.6023579084001717e-07,
+      "loss": 0.0319,
+      "reward": 2.733997344970703,
+      "reward_std": 0.4519456923007965,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.875,
+      "rewards/wrapped_driving_reward": -0.1410025954246521,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1389
+    },
+    {
+      "completion_length": 375.0,
+      "epoch": 278.0,
+      "grad_norm": 0.5273017287254333,
+      "kl": 1.0914883613586426,
+      "learning_rate": 2.5781814616827936e-07,
+      "loss": 0.0437,
+      "reward": 3.7337470054626465,
+      "reward_std": 0.15703323483467102,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9166666865348816,
+      "rewards/wrapped_driving_reward": 0.8170802593231201,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1390
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 278.2,
+      "grad_norm": 0.5100916624069214,
+      "kl": 1.3880748748779297,
+      "learning_rate": 2.5541117348685585e-07,
+      "loss": 0.0555,
+      "reward": 3.584139823913574,
+      "reward_std": 0.21088428795337677,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9375,
+      "rewards/wrapped_driving_reward": 0.646639883518219,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1391
+    },
+    {
+      "completion_length": 406.0,
+      "epoch": 278.4,
+      "grad_norm": 0.5762739181518555,
+      "kl": 0.5575605630874634,
+      "learning_rate": 2.53014884252083e-07,
+      "loss": 0.0223,
+      "reward": 1.9318915605545044,
+      "reward_std": 1.9687825441360474,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.0681085586547852,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1392
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 278.6,
+      "grad_norm": 0.47159233689308167,
+      "kl": 1.284196376800537,
+      "learning_rate": 2.506292898694468e-07,
+      "loss": 0.0514,
+      "reward": 2.107334613800049,
+      "reward_std": 2.0859498977661133,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9886363744735718,
+      "rewards/wrapped_driving_reward": -0.7563016414642334,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1393
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 278.8,
+      "grad_norm": 0.5752893090248108,
+      "kl": 0.9389250874519348,
+      "learning_rate": 2.482544016935304e-07,
+      "loss": 0.0376,
+      "reward": 2.650101661682129,
+      "reward_std": 0.5187346935272217,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9772727489471436,
+      "rewards/wrapped_driving_reward": -0.07717101275920868,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1394
+    },
+    {
+      "completion_length": 435.0,
+      "epoch": 279.0,
+      "grad_norm": 0.5396931171417236,
+      "kl": 1.2249048948287964,
+      "learning_rate": 2.458902310279601e-07,
+      "loss": 0.049,
+      "reward": 3.8143632411956787,
+      "reward_std": 0.031357116997241974,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9821428656578064,
+      "rewards/wrapped_driving_reward": 0.8322204351425171,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1395
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 279.2,
+      "grad_norm": 0.5077393054962158,
+      "kl": 1.0966877937316895,
+      "learning_rate": 2.43536789125349e-07,
+      "loss": 0.0439,
+      "reward": 2.85006046295166,
+      "reward_std": 0.4239778518676758,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.10006056725978851,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1396
+    },
+    {
+      "completion_length": 495.0,
+      "epoch": 279.4,
+      "grad_norm": 0.47476890683174133,
+      "kl": 0.7175695896148682,
+      "learning_rate": 2.411940871872454e-07,
+      "loss": 0.0287,
+      "reward": 2.7550477981567383,
+      "reward_std": 1.070473313331604,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.24495217204093933,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1397
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 279.6,
+      "grad_norm": 0.5334399342536926,
+      "kl": 0.5907444953918457,
+      "learning_rate": 2.388621363640797e-07,
+      "loss": 0.0236,
+      "reward": 1.995520830154419,
+      "reward_std": 2.4302823543548584,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8885281085968018,
+      "rewards/wrapped_driving_reward": -0.7680072784423828,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1398
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 279.8,
+      "grad_norm": 0.48916390538215637,
+      "kl": 1.086790680885315,
+      "learning_rate": 2.365409477551117e-07,
+      "loss": 0.0435,
+      "reward": 2.6571402549743652,
+      "reward_std": 0.2852739989757538,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.0928598940372467,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1399
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 280.0,
+      "grad_norm": 0.45400363206863403,
+      "kl": 1.350327491760254,
+      "learning_rate": 2.3423053240837518e-07,
+      "loss": 0.054,
+      "reward": 2.4340896606445312,
+      "reward_std": 0.9140759706497192,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9722222089767456,
+      "rewards/wrapped_driving_reward": -0.5381323099136353,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1400
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 280.2,
+      "grad_norm": 0.5691618323326111,
+      "kl": 1.2794947624206543,
+      "learning_rate": 2.3193090132062696e-07,
+      "loss": 0.0512,
+      "reward": 2.5558643341064453,
+      "reward_std": 0.878370463848114,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.4441356658935547,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1401
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 280.4,
+      "grad_norm": 0.6462184190750122,
+      "kl": 1.1858443021774292,
+      "learning_rate": 2.2964206543729662e-07,
+      "loss": 0.0474,
+      "reward": 1.185817003250122,
+      "reward_std": 3.124349355697632,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.71875,
+      "rewards/wrapped_driving_reward": -1.157932996749878,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1402
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 280.6,
+      "grad_norm": 0.6827697157859802,
+      "kl": 1.3049417734146118,
+      "learning_rate": 2.2736403565243037e-07,
+      "loss": 0.0522,
+      "reward": 3.4060757160186768,
+      "reward_std": 0.5276473164558411,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.918749988079071,
+      "rewards/wrapped_driving_reward": 0.73732590675354,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1403
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 280.8,
+      "grad_norm": 0.5299985408782959,
+      "kl": 0.704107403755188,
+      "learning_rate": 2.2509682280864225e-07,
+      "loss": 0.0282,
+      "reward": 2.082700252532959,
+      "reward_std": 2.060072898864746,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9166666865348816,
+      "rewards/wrapped_driving_reward": -0.8339664936065674,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1404
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 281.0,
+      "grad_norm": 0.4715111255645752,
+      "kl": 0.5978248119354248,
+      "learning_rate": 2.2284043769706026e-07,
+      "loss": 0.0239,
+      "reward": 1.7607383728027344,
+      "reward_std": 3.847649335861206,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.36426153779029846,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1405
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 281.2,
+      "grad_norm": 0.4889107048511505,
+      "kl": 0.7684929966926575,
+      "learning_rate": 2.2059489105727862e-07,
+      "loss": 0.0307,
+      "reward": 1.8192955255508423,
+      "reward_std": 3.5630123615264893,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.5557044148445129,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1406
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 281.4,
+      "grad_norm": 0.49164026975631714,
+      "kl": 1.285252332687378,
+      "learning_rate": 2.1836019357730194e-07,
+      "loss": 0.0514,
+      "reward": 2.6589338779449463,
+      "reward_std": 0.2288684844970703,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.21606609225273132,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1407
+    },
+    {
+      "completion_length": 438.0,
+      "epoch": 281.6,
+      "grad_norm": 0.5161469578742981,
+      "kl": 0.7173618078231812,
+      "learning_rate": 2.1613635589349756e-07,
+      "loss": 0.0287,
+      "reward": 2.9021034240722656,
+      "reward_std": 0.9981189966201782,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.09789654612541199,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1408
+    },
+    {
+      "completion_length": 436.0,
+      "epoch": 281.8,
+      "grad_norm": 0.523916482925415,
+      "kl": 1.2481708526611328,
+      "learning_rate": 2.139233885905448e-07,
+      "loss": 0.0499,
+      "reward": 3.0168285369873047,
+      "reward_std": 0.46051692962646484,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8333333730697632,
+      "rewards/wrapped_driving_reward": 0.18349523842334747,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1409
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 282.0,
+      "grad_norm": 0.5535622835159302,
+      "kl": 0.7396218180656433,
+      "learning_rate": 2.1172130220138227e-07,
+      "loss": 0.0296,
+      "reward": 1.5402005910873413,
+      "reward_std": 3.6936464309692383,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.7097994089126587,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1410
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 282.2,
+      "grad_norm": 0.5390403270721436,
+      "kl": 0.5375873446464539,
+      "learning_rate": 2.0953010720716037e-07,
+      "loss": 0.0215,
+      "reward": 3.148954153060913,
+      "reward_std": 0.2462591975927353,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.14895422756671906,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1411
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 282.4,
+      "grad_norm": 1.2856696844100952,
+      "kl": 1.3331913948059082,
+      "learning_rate": 2.073498140371899e-07,
+      "loss": 0.0533,
+      "reward": 2.4814517498016357,
+      "reward_std": 0.4789073169231415,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9166666865348816,
+      "rewards/wrapped_driving_reward": -0.18521496653556824,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1412
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 282.6,
+      "grad_norm": 0.5040442943572998,
+      "kl": 0.9957391619682312,
+      "learning_rate": 2.0518043306889334e-07,
+      "loss": 0.0398,
+      "reward": 1.3879209756851196,
+      "reward_std": 3.283914804458618,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9870790839195251,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1413
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 282.8,
+      "grad_norm": 0.4058266878128052,
+      "kl": 1.5492812395095825,
+      "learning_rate": 2.0302197462775453e-07,
+      "loss": 0.062,
+      "reward": 3.1179046630859375,
+      "reward_std": 0.5837324261665344,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8854166865348816,
+      "rewards/wrapped_driving_reward": 0.48248809576034546,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1414
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 283.0,
+      "grad_norm": 0.5042555332183838,
+      "kl": 0.38030803203582764,
+      "learning_rate": 2.0087444898726938e-07,
+      "loss": 0.0152,
+      "reward": 0.9512829780578613,
+      "reward_std": 1.3869514465332031,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.4237169027328491,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 1415
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 283.2,
+      "grad_norm": 1.04764723777771,
+      "kl": 0.9765843152999878,
+      "learning_rate": 1.9873786636889908e-07,
+      "loss": 0.0391,
+      "reward": 1.6774747371673584,
+      "reward_std": 3.798034429550171,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.5725252628326416,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1416
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 283.4,
+      "grad_norm": 0.4755512773990631,
+      "kl": 0.7267125248908997,
+      "learning_rate": 1.9661223694201898e-07,
+      "loss": 0.0291,
+      "reward": 2.8082895278930664,
+      "reward_std": 0.360664039850235,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9722222089767456,
+      "rewards/wrapped_driving_reward": -0.0389326736330986,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1417
+    },
+    {
+      "completion_length": 375.0,
+      "epoch": 283.6,
+      "grad_norm": 0.6061931848526001,
+      "kl": 1.0483731031417847,
+      "learning_rate": 1.9449757082387083e-07,
+      "loss": 0.0419,
+      "reward": 2.6765944957733154,
+      "reward_std": 0.2802552580833435,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.19840553402900696,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1418
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 283.8,
+      "grad_norm": 0.4729617238044739,
+      "kl": 1.1111360788345337,
+      "learning_rate": 1.9239387807951482e-07,
+      "loss": 0.0444,
+      "reward": 2.268667697906494,
+      "reward_std": 2.2074902057647705,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9147727489471436,
+      "rewards/wrapped_driving_reward": -0.646105170249939,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1419
+    },
+    {
+      "completion_length": 479.0,
+      "epoch": 284.0,
+      "grad_norm": 0.5022574663162231,
+      "kl": 0.7856901288032532,
+      "learning_rate": 1.9030116872178317e-07,
+      "loss": 0.0314,
+      "reward": 2.727975845336914,
+      "reward_std": 0.886898934841156,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.2279757261276245,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 1420
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 284.2,
+      "grad_norm": 0.4967239201068878,
+      "kl": 0.9862984418869019,
+      "learning_rate": 1.8821945271122993e-07,
+      "loss": 0.0395,
+      "reward": 2.3430233001708984,
+      "reward_std": 0.4683536887168884,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.7916666269302368,
+      "rewards/wrapped_driving_reward": -0.19864338636398315,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1421
+    },
+    {
+      "completion_length": 468.0,
+      "epoch": 284.4,
+      "grad_norm": 0.536677896976471,
+      "kl": 1.1677287817001343,
+      "learning_rate": 1.8614873995608406e-07,
+      "loss": 0.0467,
+      "reward": 3.049527645111084,
+      "reward_std": 0.10643661767244339,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.049527425318956375,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1422
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 284.6,
+      "grad_norm": 0.49175751209259033,
+      "kl": 1.0547261238098145,
+      "learning_rate": 1.8408904031220476e-07,
+      "loss": 0.0422,
+      "reward": 1.546495795249939,
+      "reward_std": 3.3698854446411133,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.828504204750061,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1423
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 284.8,
+      "grad_norm": 0.5445951819419861,
+      "kl": 0.610645055770874,
+      "learning_rate": 1.8204036358303173e-07,
+      "loss": 0.0244,
+      "reward": 2.807023525238037,
+      "reward_std": 0.43106648325920105,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.3070235550403595,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 1424
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 285.0,
+      "grad_norm": 0.6382043957710266,
+      "kl": 1.192225694656372,
+      "learning_rate": 1.800027195195389e-07,
+      "loss": 0.0477,
+      "reward": 1.7614197731018066,
+      "reward_std": 3.1929991245269775,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.7385802268981934,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1425
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 285.2,
+      "grad_norm": 0.44340983033180237,
+      "kl": 1.196262240409851,
+      "learning_rate": 1.7797611782018942e-07,
+      "loss": 0.0479,
+      "reward": 2.96022891998291,
+      "reward_std": 0.1397945135831833,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9615384340286255,
+      "rewards/wrapped_driving_reward": 0.12369057536125183,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1426
+    },
+    {
+      "completion_length": 441.0,
+      "epoch": 285.4,
+      "grad_norm": 0.5231199264526367,
+      "kl": 0.7509476542472839,
+      "learning_rate": 1.7596056813088903e-07,
+      "loss": 0.03,
+      "reward": 3.6092190742492676,
+      "reward_std": 0.37520748376846313,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9125000238418579,
+      "rewards/wrapped_driving_reward": 0.8217191696166992,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1427
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 285.6,
+      "grad_norm": 0.5406193733215332,
+      "kl": 0.6207311153411865,
+      "learning_rate": 1.7395608004493886e-07,
+      "loss": 0.0248,
+      "reward": 0.963043212890625,
+      "reward_std": 3.3169634342193604,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.1619566679000854,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1428
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 285.8,
+      "grad_norm": 0.43645530939102173,
+      "kl": 0.9867162108421326,
+      "learning_rate": 1.719626631029911e-07,
+      "loss": 0.0395,
+      "reward": 2.2472116947174072,
+      "reward_std": 0.64691561460495,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9886363744735718,
+      "rewards/wrapped_driving_reward": -0.4914245903491974,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1429
+    },
+    {
+      "completion_length": 429.0,
+      "epoch": 286.0,
+      "grad_norm": 0.6518118977546692,
+      "kl": 1.2582813501358032,
+      "learning_rate": 1.699803267930039e-07,
+      "loss": 0.0503,
+      "reward": 3.266732692718506,
+      "reward_std": 0.5679041743278503,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.5167327523231506,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1430
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 286.2,
+      "grad_norm": 0.4585857093334198,
+      "kl": 0.8007975220680237,
+      "learning_rate": 1.6800908055019465e-07,
+      "loss": 0.032,
+      "reward": 1.539993166923523,
+      "reward_std": 1.7092621326446533,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.085006833076477,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1431
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 286.4,
+      "grad_norm": 0.5841385722160339,
+      "kl": 1.051182508468628,
+      "learning_rate": 1.6604893375699594e-07,
+      "loss": 0.042,
+      "reward": 3.581332206726074,
+      "reward_std": 0.28388237953186035,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.7063324451446533,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1432
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 286.6,
+      "grad_norm": 0.5661259293556213,
+      "kl": 0.6480541825294495,
+      "learning_rate": 1.6409989574301122e-07,
+      "loss": 0.0259,
+      "reward": 2.89264178276062,
+      "reward_std": 0.5626019239425659,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9166666865348816,
+      "rewards/wrapped_driving_reward": 0.22597499191761017,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1433
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 286.8,
+      "grad_norm": 0.5671570897102356,
+      "kl": 0.6507048606872559,
+      "learning_rate": 1.62161975784971e-07,
+      "loss": 0.026,
+      "reward": 2.6673572063446045,
+      "reward_std": 0.279194176197052,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.08264275640249252,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1434
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 287.0,
+      "grad_norm": 0.4990684986114502,
+      "kl": 1.1988261938095093,
+      "learning_rate": 1.602351831066862e-07,
+      "loss": 0.048,
+      "reward": 3.4052956104278564,
+      "reward_std": 0.32919541001319885,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9886363744735718,
+      "rewards/wrapped_driving_reward": 0.41665923595428467,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1435
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 287.2,
+      "grad_norm": 0.5636922717094421,
+      "kl": 0.9200805425643921,
+      "learning_rate": 1.583195268790061e-07,
+      "loss": 0.0368,
+      "reward": 3.5920863151550293,
+      "reward_std": 0.28853851556777954,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8420862555503845,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1436
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 287.4,
+      "grad_norm": 0.5285916924476624,
+      "kl": 1.0494803190231323,
+      "learning_rate": 1.5641501621977585e-07,
+      "loss": 0.042,
+      "reward": 3.18922758102417,
+      "reward_std": 0.7707717418670654,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.5642274618148804,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1437
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 287.6,
+      "grad_norm": 0.53641277551651,
+      "kl": 0.6068298816680908,
+      "learning_rate": 1.5452166019378989e-07,
+      "loss": 0.0243,
+      "reward": 2.5735340118408203,
+      "reward_std": 0.6772871017456055,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.05146618187427521,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1438
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 287.8,
+      "grad_norm": 0.5845058560371399,
+      "kl": 1.2026838064193726,
+      "learning_rate": 1.5263946781275185e-07,
+      "loss": 0.0481,
+      "reward": 2.525228261947632,
+      "reward_std": 0.49441930651664734,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.22477181255817413,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1439
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 288.0,
+      "grad_norm": 0.49796926975250244,
+      "kl": 0.6701675653457642,
+      "learning_rate": 1.507684480352292e-07,
+      "loss": 0.0268,
+      "reward": -0.5014688968658447,
+      "reward_std": 3.5172641277313232,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -2.2514688968658447,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1440
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 288.2,
+      "grad_norm": 0.5880181193351746,
+      "kl": 0.7840191125869751,
+      "learning_rate": 1.4890860976661314e-07,
+      "loss": 0.0314,
+      "reward": 2.996073007583618,
+      "reward_std": 0.4358140230178833,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.12107305228710175,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1441
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 288.4,
+      "grad_norm": 0.5443733930587769,
+      "kl": 1.0240284204483032,
+      "learning_rate": 1.4705996185907372e-07,
+      "loss": 0.041,
+      "reward": 2.6164638996124268,
+      "reward_std": 0.8353798985481262,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9318181872367859,
+      "rewards/wrapped_driving_reward": -0.31535422801971436,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1442
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 288.6,
+      "grad_norm": 0.4295240640640259,
+      "kl": 1.45102858543396,
+      "learning_rate": 1.4522251311151926e-07,
+      "loss": 0.058,
+      "reward": 2.7028818130493164,
+      "reward_std": 0.18153458833694458,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9166666865348816,
+      "rewards/wrapped_driving_reward": -0.21378470957279205,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1443
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 288.8,
+      "grad_norm": 3.0278501510620117,
+      "kl": 1.0071033239364624,
+      "learning_rate": 1.4339627226955394e-07,
+      "loss": 0.0403,
+      "reward": 1.6431341171264648,
+      "reward_std": 3.4404783248901367,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.7318658232688904,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1444
+    },
+    {
+      "completion_length": 453.0,
+      "epoch": 289.0,
+      "grad_norm": 0.49213650822639465,
+      "kl": 0.8807816505432129,
+      "learning_rate": 1.4158124802543693e-07,
+      "loss": 0.0352,
+      "reward": 3.713721752166748,
+      "reward_std": 0.21939153969287872,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8387218713760376,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1445
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 289.2,
+      "grad_norm": 0.504957377910614,
+      "kl": 1.7153831720352173,
+      "learning_rate": 1.3977744901803953e-07,
+      "loss": 0.0686,
+      "reward": 3.1461334228515625,
+      "reward_std": 0.1565794050693512,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.27113330364227295,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1446
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 289.4,
+      "grad_norm": 0.5052013993263245,
+      "kl": 0.6921107769012451,
+      "learning_rate": 1.3798488383280489e-07,
+      "loss": 0.0277,
+      "reward": 2.641937732696533,
+      "reward_std": 0.3177681565284729,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.10806213319301605,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1447
+    },
+    {
+      "completion_length": 447.0,
+      "epoch": 289.6,
+      "grad_norm": 0.4841085970401764,
+      "kl": 0.9617977738380432,
+      "learning_rate": 1.362035610017079e-07,
+      "loss": 0.0385,
+      "reward": 3.3735311031341553,
+      "reward_std": 0.25799721479415894,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.3735309839248657,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1448
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 289.8,
+      "grad_norm": 0.4838336706161499,
+      "kl": 1.2779005765914917,
+      "learning_rate": 1.344334890032134e-07,
+      "loss": 0.0511,
+      "reward": 3.1358931064605713,
+      "reward_std": 0.27882567048072815,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9166666865348816,
+      "rewards/wrapped_driving_reward": 0.4692264795303345,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1449
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 290.0,
+      "grad_norm": 0.5472686886787415,
+      "kl": 1.2251555919647217,
+      "learning_rate": 1.3267467626223606e-07,
+      "loss": 0.049,
+      "reward": 2.124551296234131,
+      "reward_std": 0.5581721663475037,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.949999988079071,
+      "rewards/wrapped_driving_reward": -0.3254486322402954,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 1450
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 290.2,
+      "grad_norm": 0.49400484561920166,
+      "kl": 0.5098161697387695,
+      "learning_rate": 1.3092713115010037e-07,
+      "loss": 0.0204,
+      "reward": 1.157588005065918,
+      "reward_std": 2.4917261600494385,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.842411994934082,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1451
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 290.4,
+      "grad_norm": 0.5542023777961731,
+      "kl": 1.4901350736618042,
+      "learning_rate": 1.291908619845017e-07,
+      "loss": 0.0596,
+      "reward": 3.041004180908203,
+      "reward_std": 0.7472724914550781,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.2910043001174927,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1452
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 290.6,
+      "grad_norm": 0.653663694858551,
+      "kl": 0.9853954315185547,
+      "learning_rate": 1.2746587702946538e-07,
+      "loss": 0.0394,
+      "reward": 3.691150188446045,
+      "reward_std": 0.20269089937210083,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.953125,
+      "rewards/wrapped_driving_reward": 0.7380251288414001,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1453
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 290.8,
+      "grad_norm": 0.6906840801239014,
+      "kl": 0.5861183404922485,
+      "learning_rate": 1.2575218449530747e-07,
+      "loss": 0.0234,
+      "reward": -0.5197847485542297,
+      "reward_std": 3.4412569999694824,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -2.269784688949585,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1454
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 291.0,
+      "grad_norm": 0.43388989567756653,
+      "kl": 1.0935958623886108,
+      "learning_rate": 1.2404979253859722e-07,
+      "loss": 0.0437,
+      "reward": 3.303497314453125,
+      "reward_std": 0.18481159210205078,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.30349719524383545,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1455
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 291.2,
+      "grad_norm": 0.5286070704460144,
+      "kl": 0.8581548929214478,
+      "learning_rate": 1.223587092621162e-07,
+      "loss": 0.0343,
+      "reward": 1.3907313346862793,
+      "reward_std": 3.275341510772705,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7250000238418579,
+      "rewards/wrapped_driving_reward": -0.8342685699462891,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1456
+    },
+    {
+      "completion_length": 381.0,
+      "epoch": 291.4,
+      "grad_norm": 0.6452928781509399,
+      "kl": 0.48162728548049927,
+      "learning_rate": 1.2067894271482084e-07,
+      "loss": 0.0193,
+      "reward": 3.6437442302703857,
+      "reward_std": 0.2513628900051117,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9444444179534912,
+      "rewards/wrapped_driving_reward": 0.8242997527122498,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1457
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 291.6,
+      "grad_norm": 0.6035467386245728,
+      "kl": 0.7211498618125916,
+      "learning_rate": 1.1901050089180411e-07,
+      "loss": 0.0288,
+      "reward": 2.556488513946533,
+      "reward_std": 0.508558988571167,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.1935114860534668,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1458
+    },
+    {
+      "completion_length": 478.0,
+      "epoch": 291.8,
+      "grad_norm": 0.5043565034866333,
+      "kl": 1.024623990058899,
+      "learning_rate": 1.1735339173425759e-07,
+      "loss": 0.041,
+      "reward": 3.272801637649536,
+      "reward_std": 0.21799972653388977,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.27280157804489136,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1459
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 292.0,
+      "grad_norm": 0.5113813877105713,
+      "kl": 0.7773527503013611,
+      "learning_rate": 1.1570762312943295e-07,
+      "loss": 0.0311,
+      "reward": 1.8861407041549683,
+      "reward_std": 3.599116086959839,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.4888591766357422,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1460
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 292.2,
+      "grad_norm": 0.4528890550136566,
+      "kl": 0.9532660841941833,
+      "learning_rate": 1.1407320291060453e-07,
+      "loss": 0.0381,
+      "reward": 3.3314685821533203,
+      "reward_std": 0.5657018423080444,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8314686417579651,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 1461
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 292.4,
+      "grad_norm": 0.42758703231811523,
+      "kl": 1.0716456174850464,
+      "learning_rate": 1.1245013885703343e-07,
+      "loss": 0.0429,
+      "reward": -0.7704377770423889,
+      "reward_std": 2.669010877609253,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -3.020437717437744,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1462
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 292.6,
+      "grad_norm": 0.5102977156639099,
+      "kl": 1.029752492904663,
+      "learning_rate": 1.1083843869392874e-07,
+      "loss": 0.0412,
+      "reward": 0.09982109069824219,
+      "reward_std": 3.2441370487213135,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -2.150178909301758,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1463
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 292.8,
+      "grad_norm": 0.4436793625354767,
+      "kl": 1.3929414749145508,
+      "learning_rate": 1.0923811009241142e-07,
+      "loss": 0.0557,
+      "reward": 2.5716094970703125,
+      "reward_std": 0.532233476638794,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.17839032411575317,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1464
+    },
+    {
+      "completion_length": 440.0,
+      "epoch": 293.0,
+      "grad_norm": 0.6582831740379333,
+      "kl": 1.0971852540969849,
+      "learning_rate": 1.0764916066947795e-07,
+      "loss": 0.0439,
+      "reward": 1.7273926734924316,
+      "reward_std": 3.491875410079956,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.6476072072982788,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1465
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 293.2,
+      "grad_norm": 0.5683333277702332,
+      "kl": 0.887220025062561,
+      "learning_rate": 1.0607159798796396e-07,
+      "loss": 0.0355,
+      "reward": 3.274254560470581,
+      "reward_std": 0.15667809545993805,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.27425453066825867,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1466
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 293.4,
+      "grad_norm": 0.5367352366447449,
+      "kl": 0.5116279125213623,
+      "learning_rate": 1.0450542955650844e-07,
+      "loss": 0.0205,
+      "reward": 2.946831464767456,
+      "reward_std": 0.5723892450332642,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.32183152437210083,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1467
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 293.6,
+      "grad_norm": 0.5117088556289673,
+      "kl": 1.1186624765396118,
+      "learning_rate": 1.0295066282951738e-07,
+      "loss": 0.0447,
+      "reward": 1.8120925426483154,
+      "reward_std": 1.8755556344985962,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.1879075765609741,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1468
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 293.8,
+      "grad_norm": 0.47914814949035645,
+      "kl": 0.3436341881752014,
+      "learning_rate": 1.0140730520712905e-07,
+      "loss": 0.0137,
+      "reward": 2.3840789794921875,
+      "reward_std": 0.5282421112060547,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9750000238418579,
+      "rewards/wrapped_driving_reward": -0.21592092514038086,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1469
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 294.0,
+      "grad_norm": 0.5376036763191223,
+      "kl": 1.089373230934143,
+      "learning_rate": 9.98753640351785e-08,
+      "loss": 0.0436,
+      "reward": 0.1641036868095398,
+      "reward_std": 4.230945110321045,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.5858962535858154,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1470
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 294.2,
+      "grad_norm": 0.4634571671485901,
+      "kl": 1.0996878147125244,
+      "learning_rate": 9.835484660516203e-08,
+      "loss": 0.044,
+      "reward": 2.968125343322754,
+      "reward_std": 0.47963660955429077,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.21812523901462555,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1471
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 294.4,
+      "grad_norm": 0.6054075360298157,
+      "kl": 0.5581721663475037,
+      "learning_rate": 9.684576015420277e-08,
+      "loss": 0.0223,
+      "reward": 1.4924763441085815,
+      "reward_std": 3.3292739391326904,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7142857313156128,
+      "rewards/wrapped_driving_reward": -0.8468095064163208,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1472
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 294.6,
+      "grad_norm": 0.4737394154071808,
+      "kl": 0.7070013880729675,
+      "learning_rate": 9.534811186501819e-08,
+      "loss": 0.0283,
+      "reward": 2.6901955604553223,
+      "reward_std": 0.27029547095298767,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.18480457365512848,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1473
+    },
+    {
+      "completion_length": 422.0,
+      "epoch": 294.8,
+      "grad_norm": 0.5297691226005554,
+      "kl": 0.937751054763794,
+      "learning_rate": 9.386190886588208e-08,
+      "loss": 0.0375,
+      "reward": 2.1265289783477783,
+      "reward_std": 3.417825937271118,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.37347114086151123,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1474
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 295.0,
+      "grad_norm": 0.5374232530593872,
+      "kl": 1.317732572555542,
+      "learning_rate": 9.238715823059324e-08,
+      "loss": 0.0527,
+      "reward": 3.3921566009521484,
+      "reward_std": 0.5147265195846558,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9375,
+      "rewards/wrapped_driving_reward": 0.45465680956840515,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1475
+    },
+    {
+      "completion_length": 375.0,
+      "epoch": 295.2,
+      "grad_norm": 0.6615691781044006,
+      "kl": 0.6711862683296204,
+      "learning_rate": 9.092386697844264e-08,
+      "loss": 0.0268,
+      "reward": 2.8645706176757812,
+      "reward_std": 0.12552446126937866,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.1354295164346695,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1476
+    },
+    {
+      "completion_length": 413.0,
+      "epoch": 295.4,
+      "grad_norm": 0.47133728861808777,
+      "kl": 0.8530063033103943,
+      "learning_rate": 8.947204207417681e-08,
+      "loss": 0.0341,
+      "reward": 1.3502864837646484,
+      "reward_std": 1.8075850009918213,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.6497135162353516,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1477
+    },
+    {
+      "completion_length": 440.0,
+      "epoch": 295.6,
+      "grad_norm": 0.5880916118621826,
+      "kl": 0.7528977394104004,
+      "learning_rate": 8.803169042796766e-08,
+      "loss": 0.0301,
+      "reward": 3.213520050048828,
+      "reward_std": 0.845450758934021,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.875,
+      "rewards/wrapped_driving_reward": 0.4635199308395386,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1478
+    },
+    {
+      "completion_length": 473.0,
+      "epoch": 295.8,
+      "grad_norm": 0.5115828514099121,
+      "kl": 0.840237557888031,
+      "learning_rate": 8.660281889537797e-08,
+      "loss": 0.0336,
+      "reward": 2.5605483055114746,
+      "reward_std": 0.6778362989425659,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9722222089767456,
+      "rewards/wrapped_driving_reward": -0.4116739332675934,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1479
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 296.0,
+      "grad_norm": 0.5121198892593384,
+      "kl": 1.0612808465957642,
+      "learning_rate": 8.518543427732951e-08,
+      "loss": 0.0425,
+      "reward": 1.588546872138977,
+      "reward_std": 3.061115264892578,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.6614530682563782,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1480
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 296.2,
+      "grad_norm": 0.46754106879234314,
+      "kl": 1.1329292058944702,
+      "learning_rate": 8.377954332007027e-08,
+      "loss": 0.0453,
+      "reward": 2.7073209285736084,
+      "reward_std": 0.34448838233947754,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.987500011920929,
+      "rewards/wrapped_driving_reward": -0.15517908334732056,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1481
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 296.4,
+      "grad_norm": 0.4356294870376587,
+      "kl": 0.7918063402175903,
+      "learning_rate": 8.238515271514231e-08,
+      "loss": 0.0317,
+      "reward": 3.2067975997924805,
+      "reward_std": 0.2621710002422333,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.20679756999015808,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1482
+    },
+    {
+      "completion_length": 387.0,
+      "epoch": 296.6,
+      "grad_norm": 0.8215523958206177,
+      "kl": 0.7944892048835754,
+      "learning_rate": 8.100226909935061e-08,
+      "loss": 0.0318,
+      "reward": 2.3340466022491455,
+      "reward_std": 2.2566335201263428,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.41595327854156494,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1483
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 296.8,
+      "grad_norm": 0.49796605110168457,
+      "kl": 0.8012723922729492,
+      "learning_rate": 7.963089905473093e-08,
+      "loss": 0.0321,
+      "reward": 0.7435674667358398,
+      "reward_std": 2.124368906021118,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -2.00643253326416,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 1484
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 297.0,
+      "grad_norm": 0.5388270616531372,
+      "kl": 1.1254199743270874,
+      "learning_rate": 7.827104910851729e-08,
+      "loss": 0.045,
+      "reward": 3.817746639251709,
+      "reward_std": 0.04738998040556908,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9772727489471436,
+      "rewards/wrapped_driving_reward": 0.8404738903045654,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1485
+    },
+    {
+      "completion_length": 391.0,
+      "epoch": 297.2,
+      "grad_norm": 0.5300952196121216,
+      "kl": 0.8977944850921631,
+      "learning_rate": 7.692272573311427e-08,
+      "loss": 0.0359,
+      "reward": 2.4890079498291016,
+      "reward_std": 0.4786875545978546,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.13599203526973724,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1486
+    },
+    {
+      "completion_length": 290.0,
+      "epoch": 297.4,
+      "grad_norm": 0.7146153450012207,
+      "kl": 1.3597002029418945,
+      "learning_rate": 7.558593534606196e-08,
+      "loss": 0.0544,
+      "reward": 3.7465343475341797,
+      "reward_std": 0.034208670258522034,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.7465343475341797,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1487
+    },
+    {
+      "completion_length": 475.0,
+      "epoch": 297.6,
+      "grad_norm": 0.526949942111969,
+      "kl": 0.707181990146637,
+      "learning_rate": 7.426068431000883e-08,
+      "loss": 0.0283,
+      "reward": -0.43304693698883057,
+      "reward_std": 2.172607183456421,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -2.933046817779541,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 1488
+    },
+    {
+      "completion_length": 494.0,
+      "epoch": 297.8,
+      "grad_norm": 0.7148928642272949,
+      "kl": 1.0888880491256714,
+      "learning_rate": 7.294697893267977e-08,
+      "loss": 0.0436,
+      "reward": 3.7778046131134033,
+      "reward_std": 0.09218687564134598,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9545454382896423,
+      "rewards/wrapped_driving_reward": 0.8232591152191162,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1489
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 298.0,
+      "grad_norm": 0.5720317959785461,
+      "kl": 0.7615952491760254,
+      "learning_rate": 7.164482546684642e-08,
+      "loss": 0.0305,
+      "reward": 2.964923858642578,
+      "reward_std": 0.5203008055686951,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.08992405235767365,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1490
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 298.2,
+      "grad_norm": 0.826501727104187,
+      "kl": 0.2857447564601898,
+      "learning_rate": 7.035423011029719e-08,
+      "loss": 0.0114,
+      "reward": 2.1296918392181396,
+      "reward_std": 0.7635976076126099,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.37030819058418274,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 1491
+    },
+    {
+      "completion_length": 447.0,
+      "epoch": 298.4,
+      "grad_norm": 0.6443661451339722,
+      "kl": 0.9502916932106018,
+      "learning_rate": 6.907519900580862e-08,
+      "loss": 0.038,
+      "reward": 3.5938684940338135,
+      "reward_std": 0.2614801228046417,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.7188684940338135,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1492
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 298.6,
+      "grad_norm": 0.5429460406303406,
+      "kl": 0.5413885116577148,
+      "learning_rate": 6.780773824111436e-08,
+      "loss": 0.0217,
+      "reward": 2.940061569213867,
+      "reward_std": 0.5089468359947205,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.3150615990161896,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1493
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 298.8,
+      "grad_norm": 0.5635232329368591,
+      "kl": 0.5069034695625305,
+      "learning_rate": 6.65518538488788e-08,
+      "loss": 0.0203,
+      "reward": 2.2883620262145996,
+      "reward_std": 0.4040224254131317,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.3366377651691437,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1494
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 299.0,
+      "grad_norm": 0.519482433795929,
+      "kl": 0.7159340381622314,
+      "learning_rate": 6.530755180666593e-08,
+      "loss": 0.0286,
+      "reward": 3.293801784515381,
+      "reward_std": 0.2992297112941742,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9255952835083008,
+      "rewards/wrapped_driving_reward": 0.49320662021636963,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1495
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 299.2,
+      "grad_norm": 0.40668848156929016,
+      "kl": 1.481412649154663,
+      "learning_rate": 6.407483803691216e-08,
+      "loss": 0.0593,
+      "reward": 2.267841100692749,
+      "reward_std": 2.2208962440490723,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8999999761581421,
+      "rewards/wrapped_driving_reward": -0.6321589946746826,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1496
+    },
+    {
+      "completion_length": 495.0,
+      "epoch": 299.4,
+      "grad_norm": 0.5128720998764038,
+      "kl": 0.7865632772445679,
+      "learning_rate": 6.285371840689725e-08,
+      "loss": 0.0315,
+      "reward": 2.8370330333709717,
+      "reward_std": 0.678887128829956,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.925000011920929,
+      "rewards/wrapped_driving_reward": -0.08796697854995728,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 1497
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 299.6,
+      "grad_norm": 0.6974601745605469,
+      "kl": 0.44543877243995667,
+      "learning_rate": 6.164419872871835e-08,
+      "loss": 0.0178,
+      "reward": 2.830705165863037,
+      "reward_std": 0.21339364349842072,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9772727489471436,
+      "rewards/wrapped_driving_reward": 0.22843225300312042,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1498
+    },
+    {
+      "completion_length": 487.0,
+      "epoch": 299.8,
+      "grad_norm": 0.5417752861976624,
+      "kl": 1.1540416479110718,
+      "learning_rate": 6.044628475925962e-08,
+      "loss": 0.0462,
+      "reward": 2.005797863006592,
+      "reward_std": 3.6705517768859863,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.3692021667957306,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 1499
+    },
+    {
+      "completion_length": 500.0,
+      "epoch": 300.0,
+      "grad_norm": 0.47423315048217773,
+      "kl": 0.6840797662734985,
+      "learning_rate": 5.92599822001666e-08,
+      "loss": 0.0274,
+      "reward": 1.5093450546264648,
+      "reward_std": 1.8018314838409424,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.1156549453735352,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 1500
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 1600,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 320,
+  "save_steps": 250,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}