diff --git "a/checkpoint-750/trainer_state.json" "b/checkpoint-750/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/checkpoint-750/trainer_state.json"
@@ -0,0 +1,11284 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 30.0,
+  "eval_steps": 500,
+  "global_step": 750,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "completion_length": 750.0,
+      "epoch": 0.04,
+      "grad_norm": 12918.7890625,
+      "kl": 1354.3233642578125,
+      "learning_rate": 3.1250000000000005e-08,
+      "loss": 54.1729,
+      "reward": 0.7556291222572327,
+      "reward_std": 3.186340093612671,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9943709373474121,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 1
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 0.08,
+      "grad_norm": 118175080.0,
+      "kl": 11214800.0,
+      "learning_rate": 6.250000000000001e-08,
+      "loss": 448592.0,
+      "reward": -0.4338679313659668,
+      "reward_std": 4.129403591156006,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.6838679313659668,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 2
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 0.12,
+      "grad_norm": 31.031461715698242,
+      "kl": 11.660053253173828,
+      "learning_rate": 9.375e-08,
+      "loss": 0.4664,
+      "reward": 2.544208526611328,
+      "reward_std": 0.4753165543079376,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.16920866072177887,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 3
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 0.16,
+      "grad_norm": 44.48072814941406,
+      "kl": 10.989585876464844,
+      "learning_rate": 1.2500000000000002e-07,
+      "loss": 0.4396,
+      "reward": 0.6104838252067566,
+      "reward_std": 2.7562637329101562,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.1395162343978882,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 4
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 0.2,
+      "grad_norm": 177.81173706054688,
+      "kl": 29.71923065185547,
+      "learning_rate": 1.5625e-07,
+      "loss": 1.1888,
+      "reward": 0.7251285314559937,
+      "reward_std": 3.179880380630493,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.0248714685440063,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 5
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 0.24,
+      "grad_norm": 538.7103881835938,
+      "kl": 77.67715454101562,
+      "learning_rate": 1.875e-07,
+      "loss": 3.1071,
+      "reward": 1.9332659244537354,
+      "reward_std": 3.2915334701538086,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.44173407554626465,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 6
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 0.28,
+      "grad_norm": 1767973.125,
+      "kl": 117501.96875,
+      "learning_rate": 2.1875e-07,
+      "loss": 4700.0791,
+      "reward": -2.279214859008789,
+      "reward_std": 3.441570281982422,
+      "rewards/mpc_param_extraction_reward": 0.25,
+      "rewards/mpc_param_name_reward": 0.25,
+      "rewards/wrapped_driving_reward": -2.779214859008789,
+      "rewards/wrapped_format_reward": 0.0,
+      "step": 7
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 0.32,
+      "grad_norm": 43.647186279296875,
+      "kl": 15.320382118225098,
+      "learning_rate": 2.5000000000000004e-07,
+      "loss": 0.6128,
+      "reward": 0.5996664762496948,
+      "reward_std": 3.1008288860321045,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.625,
+      "rewards/wrapped_driving_reward": -0.9003335237503052,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 8
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 0.36,
+      "grad_norm": 913985.875,
+      "kl": 96429.6015625,
+      "learning_rate": 2.8125e-07,
+      "loss": 3857.1833,
+      "reward": -0.5126274824142456,
+      "reward_std": 3.7559752464294434,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -2.137627601623535,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 9
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 0.4,
+      "grad_norm": 15.540242195129395,
+      "kl": 5.14943265914917,
+      "learning_rate": 3.125e-07,
+      "loss": 0.206,
+      "reward": 0.7949561476707458,
+      "reward_std": 3.2785708904266357,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.8300438523292542,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 10
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 0.44,
+      "grad_norm": 30.965370178222656,
+      "kl": 9.962629318237305,
+      "learning_rate": 3.4375000000000004e-07,
+      "loss": 0.3985,
+      "reward": -1.625,
+      "reward_std": 0.25,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 11
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 0.48,
+      "grad_norm": 66.62539672851562,
+      "kl": 12.977354049682617,
+      "learning_rate": 3.75e-07,
+      "loss": 0.5191,
+      "reward": -2.25,
+      "reward_std": 1.1902379989624023,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 12
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 0.52,
+      "grad_norm": 5716130.0,
+      "kl": 649889.6875,
+      "learning_rate": 4.0625000000000003e-07,
+      "loss": 25995.5879,
+      "reward": -3.875,
+      "reward_std": 0.25,
+      "rewards/mpc_param_extraction_reward": 0.0,
+      "rewards/mpc_param_name_reward": 0.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 13
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 0.56,
+      "grad_norm": 87.81452941894531,
+      "kl": 14.151000022888184,
+      "learning_rate": 4.375e-07,
+      "loss": 0.566,
+      "reward": -2.25,
+      "reward_std": 1.1902379989624023,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 14
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 0.6,
+      "grad_norm": 573.2177124023438,
+      "kl": 77.2968521118164,
+      "learning_rate": 4.6875000000000006e-07,
+      "loss": 3.0919,
+      "reward": -0.5207003355026245,
+      "reward_std": 4.02908182144165,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.7707003355026245,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 15
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 0.64,
+      "grad_norm": 691142.75,
+      "kl": 47404.98828125,
+      "learning_rate": 5.000000000000001e-07,
+      "loss": 1896.199,
+      "reward": -0.6887123584747314,
+      "reward_std": 3.8238091468811035,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -2.0637123584747314,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 16
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 0.68,
+      "grad_norm": 54.24480056762695,
+      "kl": 13.760282516479492,
+      "learning_rate": 5.3125e-07,
+      "loss": 0.5504,
+      "reward": -0.7901134490966797,
+      "reward_std": 3.464918375015259,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -2.0401134490966797,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 17
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 0.72,
+      "grad_norm": 2934.07861328125,
+      "kl": 389.90374755859375,
+      "learning_rate": 5.625e-07,
+      "loss": 15.5961,
+      "reward": 0.821560800075531,
+      "reward_std": 3.2411201000213623,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.6785714626312256,
+      "rewards/wrapped_driving_reward": -0.9820106625556946,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 18
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 0.76,
+      "grad_norm": 6.749892234802246,
+      "kl": 2.5047595500946045,
+      "learning_rate": 5.9375e-07,
+      "loss": 0.1002,
+      "reward": 2.3981642723083496,
+      "reward_std": 0.7526259422302246,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.14816424250602722,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 19
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 0.8,
+      "grad_norm": 2.598149061203003,
+      "kl": 1.0905311107635498,
+      "learning_rate": 6.25e-07,
+      "loss": 0.0436,
+      "reward": 1.3226027488708496,
+      "reward_std": 2.890576124191284,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.5523972511291504,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 20
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 0.84,
+      "grad_norm": 19.09586524963379,
+      "kl": 5.524669170379639,
+      "learning_rate": 6.562500000000001e-07,
+      "loss": 0.221,
+      "reward": 2.1500957012176514,
+      "reward_std": 0.18781162798404694,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.09990427643060684,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 21
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 0.88,
+      "grad_norm": 20067694.0,
+      "kl": 1812346.0,
+      "learning_rate": 6.875000000000001e-07,
+      "loss": 72493.8359,
+      "reward": -0.6700150966644287,
+      "reward_std": 3.845890998840332,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.7950150966644287,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 22
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 0.92,
+      "grad_norm": 23.432483673095703,
+      "kl": 6.9398393630981445,
+      "learning_rate": 7.1875e-07,
+      "loss": 0.2776,
+      "reward": -0.5996897220611572,
+      "reward_std": 1.5026532411575317,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -2.974689483642578,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 23
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 0.96,
+      "grad_norm": 13.615424156188965,
+      "kl": 2.8062376976013184,
+      "learning_rate": 7.5e-07,
+      "loss": 0.1122,
+      "reward": -1.6607142686843872,
+      "reward_std": 0.47155481576919556,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9642857313156128,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 24
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 1.0,
+      "grad_norm": 630.5245361328125,
+      "kl": 71.51762390136719,
+      "learning_rate": 7.8125e-07,
+      "loss": 2.8607,
+      "reward": -0.30632930994033813,
+      "reward_std": 2.6143534183502197,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -2.1813292503356934,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 25
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 1.04,
+      "grad_norm": 5.207806587219238,
+      "kl": 2.9511070251464844,
+      "learning_rate": 8.125000000000001e-07,
+      "loss": 0.118,
+      "reward": 3.2842254638671875,
+      "reward_std": 0.35285714268684387,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.7842254638671875,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 26
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 1.08,
+      "grad_norm": 30.472753524780273,
+      "kl": 9.41655445098877,
+      "learning_rate": 8.437500000000001e-07,
+      "loss": 0.3767,
+      "reward": 0.07714283466339111,
+      "reward_std": 2.906501293182373,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.9228571653366089,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 27
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 1.12,
+      "grad_norm": 16.97286033630371,
+      "kl": 7.253666400909424,
+      "learning_rate": 8.75e-07,
+      "loss": 0.2901,
+      "reward": -0.07512685656547546,
+      "reward_std": 2.7640981674194336,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.8251267671585083,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 28
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 1.16,
+      "grad_norm": 1228.613037109375,
+      "kl": 160.93626403808594,
+      "learning_rate": 9.0625e-07,
+      "loss": 6.4375,
+      "reward": -1.879897117614746,
+      "reward_std": 2.8305463790893555,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -3.254897117614746,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 29
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 1.2,
+      "grad_norm": 633888.125,
+      "kl": 70588.515625,
+      "learning_rate": 9.375000000000001e-07,
+      "loss": 2823.5408,
+      "reward": -2.25,
+      "reward_std": 1.2583057880401611,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 30
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 1.24,
+      "grad_norm": 62.07278060913086,
+      "kl": 7.735680103302002,
+      "learning_rate": 9.6875e-07,
+      "loss": 0.3094,
+      "reward": 0.7910268306732178,
+      "reward_std": 3.1987111568450928,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9589731693267822,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 31
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 1.28,
+      "grad_norm": 10.78812026977539,
+      "kl": 3.6586508750915527,
+      "learning_rate": 1.0000000000000002e-06,
+      "loss": 0.1463,
+      "reward": 3.0689785480499268,
+      "reward_std": 0.5278481245040894,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.6939785480499268,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 32
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 1.32,
+      "grad_norm": 16.368228912353516,
+      "kl": 5.255997180938721,
+      "learning_rate": 1.03125e-06,
+      "loss": 0.2102,
+      "reward": 1.9649852514266968,
+      "reward_std": 1.2069640159606934,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.41001471877098083,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 33
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 1.3599999999999999,
+      "grad_norm": 4.65078067779541,
+      "kl": 2.813537120819092,
+      "learning_rate": 1.0625e-06,
+      "loss": 0.1125,
+      "reward": 1.057879090309143,
+      "reward_std": 3.4074575901031494,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9421209096908569,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 34
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 1.4,
+      "grad_norm": 8.35409164428711,
+      "kl": 4.364602565765381,
+      "learning_rate": 1.0937500000000001e-06,
+      "loss": 0.1746,
+      "reward": 0.4969135522842407,
+      "reward_std": 3.1264455318450928,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.2530864477157593,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 35
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 1.44,
+      "grad_norm": 22052300.0,
+      "kl": 1883969.0,
+      "learning_rate": 1.125e-06,
+      "loss": 75358.7578,
+      "reward": 3.022550106048584,
+      "reward_std": 0.5745974183082581,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.6475501656532288,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 36
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 1.48,
+      "grad_norm": 3.6894285678863525,
+      "kl": 2.3495442867279053,
+      "learning_rate": 1.1562500000000002e-06,
+      "loss": 0.094,
+      "reward": 2.525552749633789,
+      "reward_std": 0.5908641219139099,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.025552626699209213,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 37
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 1.52,
+      "grad_norm": 35878.66796875,
+      "kl": 4133.42822265625,
+      "learning_rate": 1.1875e-06,
+      "loss": 165.3371,
+      "reward": 2.06065034866333,
+      "reward_std": 0.20606659352779388,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9583333134651184,
+      "rewards/wrapped_driving_reward": -0.02268284745514393,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 38
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 1.56,
+      "grad_norm": 24.238332748413086,
+      "kl": 6.0253143310546875,
+      "learning_rate": 1.21875e-06,
+      "loss": 0.241,
+      "reward": 0.5050567388534546,
+      "reward_std": 2.368363380432129,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -2.119943380355835,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 39
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 1.6,
+      "grad_norm": 9.88853931427002,
+      "kl": 2.069253921508789,
+      "learning_rate": 1.25e-06,
+      "loss": 0.0828,
+      "reward": -0.38425058126449585,
+      "reward_std": 4.177649021148682,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.634250521659851,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 40
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 1.6400000000000001,
+      "grad_norm": 245.3301239013672,
+      "kl": 16.598329544067383,
+      "learning_rate": 1.28125e-06,
+      "loss": 0.6639,
+      "reward": 0.8135783672332764,
+      "reward_std": 3.220566987991333,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9364216327667236,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 41
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 1.6800000000000002,
+      "grad_norm": 7.694745063781738,
+      "kl": 3.324167490005493,
+      "learning_rate": 1.3125000000000001e-06,
+      "loss": 0.133,
+      "reward": 0.5644169449806213,
+      "reward_std": 3.078721523284912,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.0605831146240234,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 42
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 1.72,
+      "grad_norm": 427072.96875,
+      "kl": 22608.654296875,
+      "learning_rate": 1.34375e-06,
+      "loss": 904.3459,
+      "reward": -2.875,
+      "reward_std": 1.314977765083313,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 43
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 1.76,
+      "grad_norm": 169.96109008789062,
+      "kl": 13.643832206726074,
+      "learning_rate": 1.3750000000000002e-06,
+      "loss": 0.5458,
+      "reward": -2.75,
+      "reward_std": 1.1902379989624023,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 44
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 1.8,
+      "grad_norm": 1.771145224571228,
+      "kl": 1.1954182386398315,
+      "learning_rate": 1.40625e-06,
+      "loss": 0.0478,
+      "reward": -1.5,
+      "reward_std": 0.40824830532073975,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 45
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 1.8399999999999999,
+      "grad_norm": 92.88534545898438,
+      "kl": 12.012389183044434,
+      "learning_rate": 1.4375e-06,
+      "loss": 0.4805,
+      "reward": 1.650305986404419,
+      "reward_std": 2.439289093017578,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.8496940732002258,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 46
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 1.88,
+      "grad_norm": 82.52506256103516,
+      "kl": 9.6003999710083,
+      "learning_rate": 1.4687500000000001e-06,
+      "loss": 0.384,
+      "reward": 2.39274001121521,
+      "reward_std": 0.08233900368213654,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9615384340286255,
+      "rewards/wrapped_driving_reward": 0.05620140582323074,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 47
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 1.92,
+      "grad_norm": 163.56568908691406,
+      "kl": 20.40851593017578,
+      "learning_rate": 1.5e-06,
+      "loss": 0.8163,
+      "reward": 2.7317166328430176,
+      "reward_std": 0.4628515839576721,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.606716513633728,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 48
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 1.96,
+      "grad_norm": 7.335759162902832,
+      "kl": 2.264324426651001,
+      "learning_rate": 1.5312500000000002e-06,
+      "loss": 0.0906,
+      "reward": -1.90625,
+      "reward_std": 0.1875,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.96875,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 49
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 2.0,
+      "grad_norm": 157.92581176757812,
+      "kl": 25.004880905151367,
+      "learning_rate": 1.5625e-06,
+      "loss": 1.0002,
+      "reward": -2.5,
+      "reward_std": 1.7320507764816284,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 50
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 2.04,
+      "grad_norm": 173265.328125,
+      "kl": 19213.73828125,
+      "learning_rate": 1.59375e-06,
+      "loss": 768.5495,
+      "reward": -2.8977272510528564,
+      "reward_std": 1.295454502105713,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.47727274894714355,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 51
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 2.08,
+      "grad_norm": 94274.2890625,
+      "kl": 11336.91015625,
+      "learning_rate": 1.6250000000000001e-06,
+      "loss": 453.4765,
+      "reward": -0.661945104598999,
+      "reward_std": 3.859673023223877,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.6619449853897095,
+      "rewards/wrapped_format_reward": 0.0,
+      "step": 52
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 2.12,
+      "grad_norm": 8.44528579711914,
+      "kl": 3.5165162086486816,
+      "learning_rate": 1.65625e-06,
+      "loss": 0.1407,
+      "reward": 0.029820501804351807,
+      "reward_std": 1.8348188400268555,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -2.345179557800293,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 53
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 2.16,
+      "grad_norm": 847.0797729492188,
+      "kl": 106.91917419433594,
+      "learning_rate": 1.6875000000000001e-06,
+      "loss": 4.2768,
+      "reward": 1.4347769021987915,
+      "reward_std": 3.6399362087249756,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.4402230978012085,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 54
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 2.2,
+      "grad_norm": 2.3560614585876465,
+      "kl": 1.385076642036438,
+      "learning_rate": 1.71875e-06,
+      "loss": 0.0554,
+      "reward": 2.0996251106262207,
+      "reward_std": 1.1530507802963257,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.6503750085830688,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 55
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 2.24,
+      "grad_norm": 21.23666000366211,
+      "kl": 5.022896766662598,
+      "learning_rate": 1.75e-06,
+      "loss": 0.2009,
+      "reward": 0.9647395610809326,
+      "reward_std": 3.323491334915161,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.5352604389190674,
+      "rewards/wrapped_format_reward": 0.0,
+      "step": 56
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 2.2800000000000002,
+      "grad_norm": 5.362183570861816,
+      "kl": 1.007752776145935,
+      "learning_rate": 1.78125e-06,
+      "loss": 0.0403,
+      "reward": 2.879687786102295,
+      "reward_std": 0.23178231716156006,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.6296878457069397,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 57
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 2.32,
+      "grad_norm": 18.63233184814453,
+      "kl": 5.4822258949279785,
+      "learning_rate": 1.8125e-06,
+      "loss": 0.2193,
+      "reward": 2.6016368865966797,
+      "reward_std": 0.5958145260810852,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.22663694620132446,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 58
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 2.36,
+      "grad_norm": 7.565363883972168,
+      "kl": 1.224977970123291,
+      "learning_rate": 1.8437500000000003e-06,
+      "loss": 0.049,
+      "reward": -1.625,
+      "reward_std": 0.4787135720252991,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 59
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 2.4,
+      "grad_norm": 1.9011263847351074,
+      "kl": 1.0183587074279785,
+      "learning_rate": 1.8750000000000003e-06,
+      "loss": 0.0407,
+      "reward": 2.4753334522247314,
+      "reward_std": 0.7654703855514526,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.024666596204042435,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 60
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 2.44,
+      "grad_norm": 10.706725120544434,
+      "kl": 0.6977672576904297,
+      "learning_rate": 1.90625e-06,
+      "loss": 0.0279,
+      "reward": -1.615384578704834,
+      "reward_std": 0.46895742416381836,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.884615421295166,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 61
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 2.48,
+      "grad_norm": 4.232525825500488,
+      "kl": 1.5266469717025757,
+      "learning_rate": 1.9375e-06,
+      "loss": 0.0611,
+      "reward": 1.7749261856079102,
+      "reward_std": 0.21998588740825653,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.4750739336013794,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 62
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 2.52,
+      "grad_norm": 1.8051197528839111,
+      "kl": 0.7413498163223267,
+      "learning_rate": 1.96875e-06,
+      "loss": 0.0297,
+      "reward": 2.4623327255249023,
+      "reward_std": 0.44108253717422485,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.037667326629161835,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 63
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 2.56,
+      "grad_norm": 1.0843530893325806,
+      "kl": 0.8077827095985413,
+      "learning_rate": 2.0000000000000003e-06,
+      "loss": 0.0323,
+      "reward": 0.8493964672088623,
+      "reward_std": 1.4425324201583862,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.4006034135818481,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 64
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 2.6,
+      "grad_norm": 79322.578125,
+      "kl": 9403.984375,
+      "learning_rate": 2.0312500000000002e-06,
+      "loss": 376.1593,
+      "reward": -0.5744374394416809,
+      "reward_std": 3.084221124649048,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -2.5744376182556152,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 65
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 2.64,
+      "grad_norm": 1.5267881155014038,
+      "kl": 1.258867621421814,
+      "learning_rate": 2.0625e-06,
+      "loss": 0.0504,
+      "reward": 2.676821231842041,
+      "reward_std": 0.3912288248538971,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.05182119458913803,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 66
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 2.68,
+      "grad_norm": 108.15802764892578,
+      "kl": 15.135726928710938,
+      "learning_rate": 2.09375e-06,
+      "loss": 0.6054,
+      "reward": 1.7942759990692139,
+      "reward_std": 0.9946174025535583,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.7057239413261414,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 67
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 2.7199999999999998,
+      "grad_norm": 3.034290075302124,
+      "kl": 1.1447491645812988,
+      "learning_rate": 2.125e-06,
+      "loss": 0.0458,
+      "reward": 0.49094319343566895,
+      "reward_std": 3.0496573448181152,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.259056806564331,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 68
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 2.76,
+      "grad_norm": 2.874622344970703,
+      "kl": 1.35330331325531,
+      "learning_rate": 2.1562500000000003e-06,
+      "loss": 0.0541,
+      "reward": 2.652649402618408,
+      "reward_std": 0.8381170034408569,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.09735051542520523,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 69
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 2.8,
+      "grad_norm": 10.109249114990234,
+      "kl": 1.9703751802444458,
+      "learning_rate": 2.1875000000000002e-06,
+      "loss": 0.0788,
+      "reward": -1.7727272510528564,
+      "reward_std": 0.4545454680919647,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9772727489471436,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 70
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 2.84,
+      "grad_norm": 3.9002420902252197,
+      "kl": 0.9838883280754089,
+      "learning_rate": 2.21875e-06,
+      "loss": 0.0394,
+      "reward": 3.0588574409484863,
+      "reward_std": 0.5791709423065186,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8088575601577759,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 71
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 2.88,
+      "grad_norm": 13.448805809020996,
+      "kl": 1.7225451469421387,
+      "learning_rate": 2.25e-06,
+      "loss": 0.0689,
+      "reward": 2.681211471557617,
+      "reward_std": 0.5453749895095825,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.05621166527271271,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 72
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 2.92,
+      "grad_norm": 1.240159273147583,
+      "kl": 0.9635999202728271,
+      "learning_rate": 2.28125e-06,
+      "loss": 0.0385,
+      "reward": 1.7876918315887451,
+      "reward_std": 0.17503832280635834,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.21230819821357727,
+      "rewards/wrapped_format_reward": 0.0,
+      "step": 73
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 2.96,
+      "grad_norm": 1.296797513961792,
+      "kl": 0.6391518115997314,
+      "learning_rate": 2.3125000000000003e-06,
+      "loss": 0.0256,
+      "reward": 2.1097092628479004,
+      "reward_std": 0.30647313594818115,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.015290826559066772,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 74
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 3.0,
+      "grad_norm": 7.442214488983154,
+      "kl": 0.5369942784309387,
+      "learning_rate": 2.3437500000000002e-06,
+      "loss": 0.0215,
+      "reward": -1.5277777910232544,
+      "reward_std": 0.4120110273361206,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9722222089767456,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 75
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 3.04,
+      "grad_norm": 1.290964961051941,
+      "kl": 0.6742348670959473,
+      "learning_rate": 2.375e-06,
+      "loss": 0.027,
+      "reward": 2.5172605514526367,
+      "reward_std": 0.3828223943710327,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.949999988079071,
+      "rewards/wrapped_driving_reward": 0.31726059317588806,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 76
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 3.08,
+      "grad_norm": 2.531158208847046,
+      "kl": 0.9481576681137085,
+      "learning_rate": 2.40625e-06,
+      "loss": 0.0379,
+      "reward": 1.286086916923523,
+      "reward_std": 3.2022881507873535,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.963913083076477,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 77
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 3.12,
+      "grad_norm": 7.368044376373291,
+      "kl": 1.763913631439209,
+      "learning_rate": 2.4375e-06,
+      "loss": 0.0706,
+      "reward": 2.8683407306671143,
+      "reward_std": 0.7174854278564453,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.6183407306671143,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 78
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 3.16,
+      "grad_norm": 1.6558523178100586,
+      "kl": 0.8889983892440796,
+      "learning_rate": 2.4687500000000003e-06,
+      "loss": 0.0356,
+      "reward": -0.9643077850341797,
+      "reward_std": 3.584562301635742,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -2.2143075466156006,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 79
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 3.2,
+      "grad_norm": 0.8536248207092285,
+      "kl": 0.5134472250938416,
+      "learning_rate": 2.5e-06,
+      "loss": 0.0205,
+      "reward": 0.7003893256187439,
+      "reward_std": 3.1346724033355713,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9246107935905457,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 80
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 3.24,
+      "grad_norm": 3.913572311401367,
+      "kl": 1.072222352027893,
+      "learning_rate": 2.53125e-06,
+      "loss": 0.0429,
+      "reward": -2.125,
+      "reward_std": 0.9464846849441528,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 81
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 3.2800000000000002,
+      "grad_norm": 0.7095546722412109,
+      "kl": 0.43932077288627625,
+      "learning_rate": 2.5625e-06,
+      "loss": 0.0176,
+      "reward": 0.578816831111908,
+      "reward_std": 2.7208759784698486,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.0461831092834473,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 82
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 3.32,
+      "grad_norm": 1.8258914947509766,
+      "kl": 0.8880151510238647,
+      "learning_rate": 2.5937500000000004e-06,
+      "loss": 0.0355,
+      "reward": -0.019976496696472168,
+      "reward_std": 1.7887709140777588,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9722222089767456,
+      "rewards/wrapped_driving_reward": -2.3671987056732178,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 83
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 3.36,
+      "grad_norm": 21.77155303955078,
+      "kl": 4.538379669189453,
+      "learning_rate": 2.6250000000000003e-06,
+      "loss": 0.1815,
+      "reward": 2.3662824630737305,
+      "reward_std": 0.3309035003185272,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.3837175965309143,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 84
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 3.4,
+      "grad_norm": 3.119682788848877,
+      "kl": 0.6306832432746887,
+      "learning_rate": 2.65625e-06,
+      "loss": 0.0252,
+      "reward": -1.765625,
+      "reward_std": 0.2718330919742584,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.984375,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 85
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 3.44,
+      "grad_norm": 0.8083566427230835,
+      "kl": 0.6314530372619629,
+      "learning_rate": 2.6875e-06,
+      "loss": 0.0253,
+      "reward": 1.092934012413025,
+      "reward_std": 3.537501335144043,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9070659875869751,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 86
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 3.48,
+      "grad_norm": 3.2034406661987305,
+      "kl": 0.4187563955783844,
+      "learning_rate": 2.71875e-06,
+      "loss": 0.0168,
+      "reward": -0.22412437200546265,
+      "reward_std": 4.363674640655518,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.5991244316101074,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 87
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 3.52,
+      "grad_norm": 5.19920015335083,
+      "kl": 0.4368354082107544,
+      "learning_rate": 2.7500000000000004e-06,
+      "loss": 0.0175,
+      "reward": 3.156670570373535,
+      "reward_std": 0.563427209854126,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.7816706299781799,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 88
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 3.56,
+      "grad_norm": 1.6918214559555054,
+      "kl": 0.6822372674942017,
+      "learning_rate": 2.7812500000000003e-06,
+      "loss": 0.0273,
+      "reward": 2.3220458030700684,
+      "reward_std": 0.5045586824417114,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.07204583287239075,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 89
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 3.6,
+      "grad_norm": 1.532529592514038,
+      "kl": 0.5102221965789795,
+      "learning_rate": 2.8125e-06,
+      "loss": 0.0204,
+      "reward": 2.4476795196533203,
+      "reward_std": 0.35076332092285156,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.05232050269842148,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 90
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 3.64,
+      "grad_norm": 2.078274726867676,
+      "kl": 0.726905107498169,
+      "learning_rate": 2.84375e-06,
+      "loss": 0.0291,
+      "reward": 2.7883927822113037,
+      "reward_std": 0.3999682664871216,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.1633928269147873,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 91
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 3.68,
+      "grad_norm": 8.694371223449707,
+      "kl": 2.171297788619995,
+      "learning_rate": 2.875e-06,
+      "loss": 0.0869,
+      "reward": 1.4226815700531006,
+      "reward_std": 2.959470272064209,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9523183703422546,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 92
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 3.7199999999999998,
+      "grad_norm": 4.719028472900391,
+      "kl": 0.4048087000846863,
+      "learning_rate": 2.9062500000000003e-06,
+      "loss": 0.0162,
+      "reward": -1.375,
+      "reward_std": 0.4787135720252991,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 93
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 3.76,
+      "grad_norm": 2.958584785461426,
+      "kl": 0.5307955741882324,
+      "learning_rate": 2.9375000000000003e-06,
+      "loss": 0.0212,
+      "reward": 3.054999828338623,
+      "reward_std": 0.5884072184562683,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8049997687339783,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 94
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 3.8,
+      "grad_norm": 3.486060380935669,
+      "kl": 1.3434487581253052,
+      "learning_rate": 2.96875e-06,
+      "loss": 0.0537,
+      "reward": 1.0327720642089844,
+      "reward_std": 3.362086057662964,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.0922279357910156,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 95
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 3.84,
+      "grad_norm": 0.23967012763023376,
+      "kl": 0.4081713855266571,
+      "learning_rate": 3e-06,
+      "loss": 0.0163,
+      "reward": -1.5,
+      "reward_std": 0.0,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 96
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 3.88,
+      "grad_norm": 1.5661548376083374,
+      "kl": 0.6307891011238098,
+      "learning_rate": 3.03125e-06,
+      "loss": 0.0252,
+      "reward": 2.490572929382324,
+      "reward_std": 0.5211818218231201,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.009427059441804886,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 97
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 3.92,
+      "grad_norm": 1.4608389139175415,
+      "kl": 0.41656869649887085,
+      "learning_rate": 3.0625000000000003e-06,
+      "loss": 0.0167,
+      "reward": -1.5714285373687744,
+      "reward_std": 0.5313312411308289,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9285714626312256,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 98
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 3.96,
+      "grad_norm": 2.444063663482666,
+      "kl": 0.5416926741600037,
+      "learning_rate": 3.0937500000000002e-06,
+      "loss": 0.0217,
+      "reward": -1.5722651481628418,
+      "reward_std": 1.5172808170318604,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -3.572265148162842,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 99
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 4.0,
+      "grad_norm": 16.588134765625,
+      "kl": 4.053507328033447,
+      "learning_rate": 3.125e-06,
+      "loss": 0.1621,
+      "reward": 0.8218609094619751,
+      "reward_std": 2.972221612930298,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.053139090538025,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 100
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 4.04,
+      "grad_norm": 2.6997458934783936,
+      "kl": 0.8300076127052307,
+      "learning_rate": 3.15625e-06,
+      "loss": 0.0332,
+      "reward": -0.017622053623199463,
+      "reward_std": 3.1953601837158203,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7142857313156128,
+      "rewards/wrapped_driving_reward": -1.731907844543457,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 101
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 4.08,
+      "grad_norm": 4.315985202789307,
+      "kl": 0.4477883577346802,
+      "learning_rate": 3.1875e-06,
+      "loss": 0.0179,
+      "reward": -0.8165792226791382,
+      "reward_std": 3.698456287384033,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.46875,
+      "rewards/wrapped_driving_reward": -2.0353293418884277,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 102
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 4.12,
+      "grad_norm": 4.780234336853027,
+      "kl": 0.9834432601928711,
+      "learning_rate": 3.2187500000000003e-06,
+      "loss": 0.0393,
+      "reward": -2.125,
+      "reward_std": 1.314977765083313,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 103
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 4.16,
+      "grad_norm": 1.8854734897613525,
+      "kl": 1.0403401851654053,
+      "learning_rate": 3.2500000000000002e-06,
+      "loss": 0.0416,
+      "reward": 2.849097728729248,
+      "reward_std": 0.5133286118507385,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.09909792244434357,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 104
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 4.2,
+      "grad_norm": 407.2416076660156,
+      "kl": 102.19829559326172,
+      "learning_rate": 3.28125e-06,
+      "loss": 4.0879,
+      "reward": 0.3436872959136963,
+      "reward_std": 2.937178373336792,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7250000238418579,
+      "rewards/wrapped_driving_reward": -1.3813128471374512,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 105
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 4.24,
+      "grad_norm": 1.6854506731033325,
+      "kl": 0.731993556022644,
+      "learning_rate": 3.3125e-06,
+      "loss": 0.0293,
+      "reward": 3.0254149436950684,
+      "reward_std": 0.7993280291557312,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.4004148244857788,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 106
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 4.28,
+      "grad_norm": 0.8972920179367065,
+      "kl": 0.429858922958374,
+      "learning_rate": 3.34375e-06,
+      "loss": 0.0172,
+      "reward": 2.303318500518799,
+      "reward_std": 0.5618811249732971,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.32168152928352356,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 107
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 4.32,
+      "grad_norm": 1.9023702144622803,
+      "kl": 0.5097759962081909,
+      "learning_rate": 3.3750000000000003e-06,
+      "loss": 0.0204,
+      "reward": 2.4162795543670654,
+      "reward_std": 0.23349861800670624,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.29127955436706543,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 108
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 4.36,
+      "grad_norm": 0.93095463514328,
+      "kl": 0.4000062942504883,
+      "learning_rate": 3.40625e-06,
+      "loss": 0.016,
+      "reward": 2.886277198791504,
+      "reward_std": 0.5716841220855713,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.26127734780311584,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 109
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 4.4,
+      "grad_norm": 0.5152439475059509,
+      "kl": 0.3538358807563782,
+      "learning_rate": 3.4375e-06,
+      "loss": 0.0142,
+      "reward": -1.625,
+      "reward_std": 0.25,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 110
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 4.44,
+      "grad_norm": 0.6482228636741638,
+      "kl": 0.4172901213169098,
+      "learning_rate": 3.46875e-06,
+      "loss": 0.0167,
+      "reward": 2.2451469898223877,
+      "reward_std": 0.4397418797016144,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.12014690786600113,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 111
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 4.48,
+      "grad_norm": 0.7293546199798584,
+      "kl": 0.37956511974334717,
+      "learning_rate": 3.5e-06,
+      "loss": 0.0152,
+      "reward": -2.038461685180664,
+      "reward_std": 0.07692313194274902,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9615384340286255,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.0,
+      "step": 112
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 4.52,
+      "grad_norm": 0.9292676448822021,
+      "kl": 0.5957732200622559,
+      "learning_rate": 3.5312500000000007e-06,
+      "loss": 0.0238,
+      "reward": 2.8711514472961426,
+      "reward_std": 0.12351223826408386,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.7461515069007874,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 113
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 4.5600000000000005,
+      "grad_norm": 0.8307209014892578,
+      "kl": 0.5839378833770752,
+      "learning_rate": 3.5625e-06,
+      "loss": 0.0234,
+      "reward": 2.626485824584961,
+      "reward_std": 0.4099518358707428,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.3764858841896057,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 114
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 4.6,
+      "grad_norm": 1.3391035795211792,
+      "kl": 0.6039181351661682,
+      "learning_rate": 3.59375e-06,
+      "loss": 0.0242,
+      "reward": 2.645888566970825,
+      "reward_std": 0.7008417248725891,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.949999988079071,
+      "rewards/wrapped_driving_reward": 0.32088854908943176,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 115
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 4.64,
+      "grad_norm": 0.5842669606208801,
+      "kl": 0.39022237062454224,
+      "learning_rate": 3.625e-06,
+      "loss": 0.0156,
+      "reward": 2.825923442840576,
+      "reward_std": 0.4296809136867523,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.20092333853244781,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 116
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 4.68,
+      "grad_norm": 0.7245670557022095,
+      "kl": 0.3917812705039978,
+      "learning_rate": 3.65625e-06,
+      "loss": 0.0157,
+      "reward": 1.6012243032455444,
+      "reward_std": 1.6945689916610718,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.7737756371498108,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 117
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 4.72,
+      "grad_norm": 0.947012722492218,
+      "kl": 0.4678252935409546,
+      "learning_rate": 3.6875000000000007e-06,
+      "loss": 0.0187,
+      "reward": 0.9042448997497559,
+      "reward_std": 2.274247169494629,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.949999988079071,
+      "rewards/wrapped_driving_reward": -1.6707550287246704,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 118
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 4.76,
+      "grad_norm": 0.9988442659378052,
+      "kl": 0.6183064579963684,
+      "learning_rate": 3.7187500000000006e-06,
+      "loss": 0.0247,
+      "reward": -2.375,
+      "reward_std": 1.108677864074707,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 119
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 4.8,
+      "grad_norm": 0.7130206823348999,
+      "kl": 0.5347681641578674,
+      "learning_rate": 3.7500000000000005e-06,
+      "loss": 0.0214,
+      "reward": -1.4166667461395264,
+      "reward_std": 0.5527708530426025,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9583333134651184,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 120
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 4.84,
+      "grad_norm": 0.627507746219635,
+      "kl": 0.4992324709892273,
+      "learning_rate": 3.78125e-06,
+      "loss": 0.02,
+      "reward": 1.178471326828003,
+      "reward_std": 3.4837844371795654,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.699999988079071,
+      "rewards/wrapped_driving_reward": -0.7715286612510681,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 121
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 4.88,
+      "grad_norm": 0.7256002426147461,
+      "kl": 0.393168568611145,
+      "learning_rate": 3.8125e-06,
+      "loss": 0.0157,
+      "reward": 2.3499269485473633,
+      "reward_std": 0.28019580245018005,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.2750731110572815,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 122
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 4.92,
+      "grad_norm": 0.9897744059562683,
+      "kl": 0.4026646018028259,
+      "learning_rate": 3.84375e-06,
+      "loss": 0.0161,
+      "reward": 0.7121872305870056,
+      "reward_std": 2.5061769485473633,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.625,
+      "rewards/wrapped_driving_reward": -1.0378127098083496,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 123
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 4.96,
+      "grad_norm": 1.374245047569275,
+      "kl": 0.4849807322025299,
+      "learning_rate": 3.875e-06,
+      "loss": 0.0194,
+      "reward": 0.6754664182662964,
+      "reward_std": 1.7975075244903564,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.8245335817337036,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 124
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 5.0,
+      "grad_norm": 152.49087524414062,
+      "kl": 41.7037239074707,
+      "learning_rate": 3.90625e-06,
+      "loss": 1.6681,
+      "reward": 2.801071882247925,
+      "reward_std": 0.3472815752029419,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.17607180774211884,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 125
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 5.04,
+      "grad_norm": 0.5340915322303772,
+      "kl": 0.3046044409275055,
+      "learning_rate": 3.9375e-06,
+      "loss": 0.0122,
+      "reward": -2.5,
+      "reward_std": 1.2247449159622192,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 126
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 5.08,
+      "grad_norm": 2.454094648361206,
+      "kl": 1.0489752292633057,
+      "learning_rate": 3.96875e-06,
+      "loss": 0.042,
+      "reward": -1.5,
+      "reward_std": 0.40824830532073975,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 127
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 5.12,
+      "grad_norm": 0.9271315932273865,
+      "kl": 0.5982043743133545,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 0.0239,
+      "reward": 2.9926440715789795,
+      "reward_std": 0.4446185231208801,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.6176440715789795,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 128
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 5.16,
+      "grad_norm": 1.4375823736190796,
+      "kl": 0.532067596912384,
+      "learning_rate": 4.031250000000001e-06,
+      "loss": 0.0213,
+      "reward": -1.75,
+      "reward_std": 0.5,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 129
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 5.2,
+      "grad_norm": 1.463436245918274,
+      "kl": 0.30711984634399414,
+      "learning_rate": 4.0625000000000005e-06,
+      "loss": 0.0123,
+      "reward": 0.6086172461509705,
+      "reward_std": 2.8491289615631104,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.6833333373069763,
+      "rewards/wrapped_driving_reward": -1.4497160911560059,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 130
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 5.24,
+      "grad_norm": 0.5281797647476196,
+      "kl": 0.3673165738582611,
+      "learning_rate": 4.09375e-06,
+      "loss": 0.0147,
+      "reward": 2.542213201522827,
+      "reward_std": 0.4680797755718231,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.08278678357601166,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 131
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 5.28,
+      "grad_norm": 1.2764793634414673,
+      "kl": 0.626862645149231,
+      "learning_rate": 4.125e-06,
+      "loss": 0.0251,
+      "reward": 2.830139636993408,
+      "reward_std": 0.4498087763786316,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8541666865348816,
+      "rewards/wrapped_driving_reward": 0.6009730100631714,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 132
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 5.32,
+      "grad_norm": 0.768198549747467,
+      "kl": 0.6894717812538147,
+      "learning_rate": 4.15625e-06,
+      "loss": 0.0276,
+      "reward": 2.4615352153778076,
+      "reward_std": 0.5109394192695618,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.03846481069922447,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 133
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 5.36,
+      "grad_norm": 1.2825806140899658,
+      "kl": 0.46785154938697815,
+      "learning_rate": 4.1875e-06,
+      "loss": 0.0187,
+      "reward": 3.512366771697998,
+      "reward_std": 0.23564468324184418,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.7623668909072876,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 134
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 5.4,
+      "grad_norm": 1.3568897247314453,
+      "kl": 0.6507589817047119,
+      "learning_rate": 4.21875e-06,
+      "loss": 0.026,
+      "reward": 0.9110469222068787,
+      "reward_std": 2.9727671146392822,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9639530181884766,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 135
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 5.44,
+      "grad_norm": 0.7310919761657715,
+      "kl": 0.5235786437988281,
+      "learning_rate": 4.25e-06,
+      "loss": 0.0209,
+      "reward": 0.7459380030632019,
+      "reward_std": 3.165867805480957,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.0040620565414429,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 136
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 5.48,
+      "grad_norm": 1.3897533416748047,
+      "kl": 0.5510402917861938,
+      "learning_rate": 4.28125e-06,
+      "loss": 0.022,
+      "reward": 3.019265651702881,
+      "reward_std": 0.16442914307117462,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.019265584647655487,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 137
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 5.52,
+      "grad_norm": 1.4078574180603027,
+      "kl": 0.49462607502937317,
+      "learning_rate": 4.312500000000001e-06,
+      "loss": 0.0198,
+      "reward": 1.5707786083221436,
+      "reward_std": 2.394756317138672,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.9292213916778564,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 138
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 5.5600000000000005,
+      "grad_norm": 1.664668083190918,
+      "kl": 0.5930428504943848,
+      "learning_rate": 4.3437500000000006e-06,
+      "loss": 0.0237,
+      "reward": 0.5512915849685669,
+      "reward_std": 2.111541986465454,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9750000238418579,
+      "rewards/wrapped_driving_reward": -1.923708438873291,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 139
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 5.6,
+      "grad_norm": 6.902396202087402,
+      "kl": 1.005578637123108,
+      "learning_rate": 4.3750000000000005e-06,
+      "loss": 0.0402,
+      "reward": 0.832996129989624,
+      "reward_std": 0.8886765837669373,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.949999988079071,
+      "rewards/wrapped_driving_reward": -1.4920037984848022,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 140
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 5.64,
+      "grad_norm": 1.0129903554916382,
+      "kl": 0.672616183757782,
+      "learning_rate": 4.40625e-06,
+      "loss": 0.0269,
+      "reward": -2.125,
+      "reward_std": 1.314977765083313,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 141
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 5.68,
+      "grad_norm": 0.5481407642364502,
+      "kl": 0.46889528632164,
+      "learning_rate": 4.4375e-06,
+      "loss": 0.0188,
+      "reward": 1.875490665435791,
+      "reward_std": 0.4471076428890228,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.3745094835758209,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 142
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 5.72,
+      "grad_norm": 0.8321424126625061,
+      "kl": 0.7620508074760437,
+      "learning_rate": 4.46875e-06,
+      "loss": 0.0305,
+      "reward": -1.625,
+      "reward_std": 0.4787135720252991,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 143
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 5.76,
+      "grad_norm": 1.097217321395874,
+      "kl": 0.5194978713989258,
+      "learning_rate": 4.5e-06,
+      "loss": 0.0208,
+      "reward": 2.9378552436828613,
+      "reward_std": 0.6964905261993408,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.3128551244735718,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 144
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 5.8,
+      "grad_norm": 1.5057528018951416,
+      "kl": 0.6985796689987183,
+      "learning_rate": 4.53125e-06,
+      "loss": 0.0279,
+      "reward": 0.8791663646697998,
+      "reward_std": 3.2669684886932373,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.1208335161209106,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 145
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 5.84,
+      "grad_norm": 0.9157974123954773,
+      "kl": 0.6999198794364929,
+      "learning_rate": 4.5625e-06,
+      "loss": 0.028,
+      "reward": 2.6941776275634766,
+      "reward_std": 0.41056010127067566,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.1941775530576706,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 146
+    },
+    {
+      "completion_length": 738.0,
+      "epoch": 5.88,
+      "grad_norm": 1.194362998008728,
+      "kl": 0.8087308406829834,
+      "learning_rate": 4.59375e-06,
+      "loss": 0.0323,
+      "reward": 1.8218350410461426,
+      "reward_std": 1.240020751953125,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.5531650185585022,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 147
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 5.92,
+      "grad_norm": 1.5525860786437988,
+      "kl": 0.5995261073112488,
+      "learning_rate": 4.625000000000001e-06,
+      "loss": 0.024,
+      "reward": 2.7626960277557373,
+      "reward_std": 0.8373459577560425,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.3876959979534149,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 148
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 5.96,
+      "grad_norm": 0.7849404811859131,
+      "kl": 0.5838685035705566,
+      "learning_rate": 4.6562500000000005e-06,
+      "loss": 0.0234,
+      "reward": 2.0091466903686523,
+      "reward_std": 0.5293838977813721,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.987500011920929,
+      "rewards/wrapped_driving_reward": -0.35335326194763184,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 149
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 6.0,
+      "grad_norm": 8.725290298461914,
+      "kl": 0.5788177847862244,
+      "learning_rate": 4.6875000000000004e-06,
+      "loss": 0.0232,
+      "reward": 2.588832378387451,
+      "reward_std": 0.47200268507003784,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.08883260190486908,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 150
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 6.04,
+      "grad_norm": 1.5502936840057373,
+      "kl": 0.7367026805877686,
+      "learning_rate": 4.71875e-06,
+      "loss": 0.0295,
+      "reward": 3.0267982482910156,
+      "reward_std": 0.3287774324417114,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.5267983675003052,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 151
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 6.08,
+      "grad_norm": 1.2154241800308228,
+      "kl": 0.6294659972190857,
+      "learning_rate": 4.75e-06,
+      "loss": 0.0252,
+      "reward": -2.125,
+      "reward_std": 1.314977765083313,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 152
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 6.12,
+      "grad_norm": 0.6927015781402588,
+      "kl": 0.466768741607666,
+      "learning_rate": 4.781250000000001e-06,
+      "loss": 0.0187,
+      "reward": 1.2741777896881104,
+      "reward_std": 3.52226185798645,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9758223295211792,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 153
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 6.16,
+      "grad_norm": 0.49416881799697876,
+      "kl": 0.3236115872859955,
+      "learning_rate": 4.8125e-06,
+      "loss": 0.0129,
+      "reward": 1.7459195852279663,
+      "reward_std": 1.0475239753723145,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.6290804147720337,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 154
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 6.2,
+      "grad_norm": 0.6880607008934021,
+      "kl": 0.4876292645931244,
+      "learning_rate": 4.84375e-06,
+      "loss": 0.0195,
+      "reward": 1.5318889617919922,
+      "reward_std": 1.4943149089813232,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9166666865348816,
+      "rewards/wrapped_driving_reward": -0.7597777247428894,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 155
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 6.24,
+      "grad_norm": 0.46220606565475464,
+      "kl": 0.39146143198013306,
+      "learning_rate": 4.875e-06,
+      "loss": 0.0157,
+      "reward": -1.8253967761993408,
+      "reward_std": 0.5452560186386108,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.6746032238006592,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 156
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 6.28,
+      "grad_norm": 0.62086421251297,
+      "kl": 0.4868091642856598,
+      "learning_rate": 4.90625e-06,
+      "loss": 0.0195,
+      "reward": 1.3802235126495361,
+      "reward_std": 3.6146950721740723,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.6197764873504639,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 157
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 6.32,
+      "grad_norm": 0.7884992957115173,
+      "kl": 0.8528112173080444,
+      "learning_rate": 4.937500000000001e-06,
+      "loss": 0.0341,
+      "reward": 0.6898324489593506,
+      "reward_std": 1.9514682292938232,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.9351675510406494,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 158
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 6.36,
+      "grad_norm": 0.5661314129829407,
+      "kl": 0.37956494092941284,
+      "learning_rate": 4.9687500000000005e-06,
+      "loss": 0.0152,
+      "reward": 1.9870556592941284,
+      "reward_std": 0.8114857077598572,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.78125,
+      "rewards/wrapped_driving_reward": -0.16919440031051636,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 159
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 6.4,
+      "grad_norm": 0.4809100925922394,
+      "kl": 0.3137115240097046,
+      "learning_rate": 5e-06,
+      "loss": 0.0125,
+      "reward": 1.137058973312378,
+      "reward_std": 3.0915822982788086,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.8629410266876221,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 160
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 6.44,
+      "grad_norm": 0.7742670774459839,
+      "kl": 0.7155295610427856,
+      "learning_rate": 4.99999405044338e-06,
+      "loss": 0.0286,
+      "reward": -0.08523339033126831,
+      "reward_std": 3.0506365299224854,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.835233449935913,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 161
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 6.48,
+      "grad_norm": 0.510188639163971,
+      "kl": 0.45238742232322693,
+      "learning_rate": 4.999976201801837e-06,
+      "loss": 0.0181,
+      "reward": 0.43522799015045166,
+      "reward_std": 3.8238766193389893,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.5647720098495483,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 162
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 6.52,
+      "grad_norm": 0.57041335105896,
+      "kl": 0.6957306861877441,
+      "learning_rate": 4.999946454160323e-06,
+      "loss": 0.0278,
+      "reward": 2.5085318088531494,
+      "reward_std": 0.6753469705581665,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.13353177905082703,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 163
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 6.5600000000000005,
+      "grad_norm": 1.1680339574813843,
+      "kl": 0.583303689956665,
+      "learning_rate": 4.9999048076604286e-06,
+      "loss": 0.0233,
+      "reward": 2.320394992828369,
+      "reward_std": 0.9881588220596313,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.984375,
+      "rewards/wrapped_driving_reward": -0.2889798581600189,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 164
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 6.6,
+      "grad_norm": 0.4880649447441101,
+      "kl": 0.5123260617256165,
+      "learning_rate": 4.999851262500375e-06,
+      "loss": 0.0205,
+      "reward": 2.9611833095550537,
+      "reward_std": 0.3708806037902832,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.08618323504924774,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 165
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 6.64,
+      "grad_norm": 0.7171524167060852,
+      "kl": 0.6323699951171875,
+      "learning_rate": 4.999785818935018e-06,
+      "loss": 0.0253,
+      "reward": 2.383418083190918,
+      "reward_std": 1.0746413469314575,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.11658180505037308,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 166
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 6.68,
+      "grad_norm": 0.8785050511360168,
+      "kl": 0.6350330114364624,
+      "learning_rate": 4.999708477275846e-06,
+      "loss": 0.0254,
+      "reward": -1.5,
+      "reward_std": 0.5773502588272095,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 167
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 6.72,
+      "grad_norm": 0.7559798359870911,
+      "kl": 0.8585944771766663,
+      "learning_rate": 4.9996192378909785e-06,
+      "loss": 0.0343,
+      "reward": 1.9524728059768677,
+      "reward_std": 1.5683073997497559,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.7975271940231323,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 168
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 6.76,
+      "grad_norm": 1.71358060836792,
+      "kl": 0.8341420292854309,
+      "learning_rate": 4.999518101205162e-06,
+      "loss": 0.0334,
+      "reward": 1.887375831604004,
+      "reward_std": 1.938693881034851,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.7376242876052856,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 169
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 6.8,
+      "grad_norm": 0.574404776096344,
+      "kl": 0.5479518175125122,
+      "learning_rate": 4.999405067699773e-06,
+      "loss": 0.0219,
+      "reward": 3.3870460987091064,
+      "reward_std": 0.16866222023963928,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.6370459198951721,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 170
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 6.84,
+      "grad_norm": 0.5955837965011597,
+      "kl": 0.3409351110458374,
+      "learning_rate": 4.99928013791281e-06,
+      "loss": 0.0136,
+      "reward": 2.3762004375457764,
+      "reward_std": 0.7474427819252014,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.1237996518611908,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 171
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 6.88,
+      "grad_norm": 0.6897561550140381,
+      "kl": 0.621300458908081,
+      "learning_rate": 4.999143312438893e-06,
+      "loss": 0.0249,
+      "reward": 1.5901850461959839,
+      "reward_std": 0.3631085157394409,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9886363744735718,
+      "rewards/wrapped_driving_reward": -1.148451328277588,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 172
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 6.92,
+      "grad_norm": 0.5545635223388672,
+      "kl": 0.6974574327468872,
+      "learning_rate": 4.998994591929266e-06,
+      "loss": 0.0279,
+      "reward": -0.6720701456069946,
+      "reward_std": 2.0031087398529053,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -3.047070026397705,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 173
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 6.96,
+      "grad_norm": 0.5723159313201904,
+      "kl": 0.5589190721511841,
+      "learning_rate": 4.998833977091783e-06,
+      "loss": 0.0224,
+      "reward": -1.25,
+      "reward_std": 0.28867512941360474,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 174
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 7.0,
+      "grad_norm": 0.6870840787887573,
+      "kl": 0.7596628665924072,
+      "learning_rate": 4.998661468690914e-06,
+      "loss": 0.0304,
+      "reward": -1.125,
+      "reward_std": 0.25,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 175
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 7.04,
+      "grad_norm": 0.5063377022743225,
+      "kl": 0.5469480156898499,
+      "learning_rate": 4.99847706754774e-06,
+      "loss": 0.0219,
+      "reward": 1.822561264038086,
+      "reward_std": 0.155478835105896,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.6774387359619141,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 176
+    },
+    {
+      "completion_length": 739.0,
+      "epoch": 7.08,
+      "grad_norm": 0.5202552080154419,
+      "kl": 0.8028308749198914,
+      "learning_rate": 4.998280774539943e-06,
+      "loss": 0.0321,
+      "reward": 3.8296477794647217,
+      "reward_std": 0.14273938536643982,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.8296477198600769,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 177
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 7.12,
+      "grad_norm": 0.5187572240829468,
+      "kl": 0.35041287541389465,
+      "learning_rate": 4.998072590601808e-06,
+      "loss": 0.014,
+      "reward": -1.875,
+      "reward_std": 0.25,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 178
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 7.16,
+      "grad_norm": 0.6030856370925903,
+      "kl": 0.27407315373420715,
+      "learning_rate": 4.9978525167242176e-06,
+      "loss": 0.011,
+      "reward": 2.659794330596924,
+      "reward_std": 0.6332101225852966,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.2847943902015686,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 179
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 7.2,
+      "grad_norm": 0.9357779622077942,
+      "kl": 0.9146249890327454,
+      "learning_rate": 4.997620553954645e-06,
+      "loss": 0.0366,
+      "reward": 2.8247017860412598,
+      "reward_std": 0.35369452834129333,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.96875,
+      "rewards/wrapped_driving_reward": -0.01904815435409546,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 180
+    },
+    {
+      "completion_length": 741.0,
+      "epoch": 7.24,
+      "grad_norm": 0.8797070980072021,
+      "kl": 0.39436712861061096,
+      "learning_rate": 4.997376703397151e-06,
+      "loss": 0.0158,
+      "reward": 2.7287731170654297,
+      "reward_std": 0.3972662091255188,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.22877322137355804,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 181
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 7.28,
+      "grad_norm": 0.4403473436832428,
+      "kl": 0.4500048756599426,
+      "learning_rate": 4.9971209662123774e-06,
+      "loss": 0.018,
+      "reward": 2.8664050102233887,
+      "reward_std": 0.34985023736953735,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.949999988079071,
+      "rewards/wrapped_driving_reward": 0.16640505194664001,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 182
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 7.32,
+      "grad_norm": 1.0706084966659546,
+      "kl": 1.0191062688827515,
+      "learning_rate": 4.996853343617542e-06,
+      "loss": 0.0408,
+      "reward": 3.047149658203125,
+      "reward_std": 0.3775829076766968,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.1721496433019638,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 183
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 7.36,
+      "grad_norm": 0.5287938714027405,
+      "kl": 0.6224220395088196,
+      "learning_rate": 4.9965738368864345e-06,
+      "loss": 0.0249,
+      "reward": 0.23351562023162842,
+      "reward_std": 2.851604700088501,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.5164843797683716,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 184
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 7.4,
+      "grad_norm": 0.672430157661438,
+      "kl": 0.666401207447052,
+      "learning_rate": 4.996282447349408e-06,
+      "loss": 0.0267,
+      "reward": -2.125,
+      "reward_std": 1.314977765083313,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 185
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 7.44,
+      "grad_norm": 0.630480170249939,
+      "kl": 0.44004517793655396,
+      "learning_rate": 4.995979176393372e-06,
+      "loss": 0.0176,
+      "reward": -1.5,
+      "reward_std": 0.5773502588272095,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 186
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 7.48,
+      "grad_norm": 0.7747740149497986,
+      "kl": 0.5829688310623169,
+      "learning_rate": 4.99566402546179e-06,
+      "loss": 0.0233,
+      "reward": 2.4287631511688232,
+      "reward_std": 0.47438302636146545,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.07123684883117676,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 187
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 7.52,
+      "grad_norm": 1.1100307703018188,
+      "kl": 0.8917567133903503,
+      "learning_rate": 4.995336996054668e-06,
+      "loss": 0.0357,
+      "reward": 2.5867562294006348,
+      "reward_std": 0.7861148118972778,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.08675637096166611,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 188
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 7.5600000000000005,
+      "grad_norm": 1.0037741661071777,
+      "kl": 1.1412770748138428,
+      "learning_rate": 4.99499808972855e-06,
+      "loss": 0.0457,
+      "reward": 2.342970609664917,
+      "reward_std": 0.25529083609580994,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.09297055006027222,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 189
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 7.6,
+      "grad_norm": 0.5152557492256165,
+      "kl": 0.46443066000938416,
+      "learning_rate": 4.994647308096509e-06,
+      "loss": 0.0186,
+      "reward": 0.16651475429534912,
+      "reward_std": 2.873490333557129,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.5834852457046509,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 190
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 7.64,
+      "grad_norm": 0.9836487174034119,
+      "kl": 0.8981544971466064,
+      "learning_rate": 4.994284652828143e-06,
+      "loss": 0.0359,
+      "reward": -1.587499976158142,
+      "reward_std": 0.480234295129776,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9125000238418579,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 191
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 7.68,
+      "grad_norm": 0.5183222889900208,
+      "kl": 0.600283145904541,
+      "learning_rate": 4.993910125649561e-06,
+      "loss": 0.024,
+      "reward": 2.7084898948669434,
+      "reward_std": 1.0434271097183228,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.04151032119989395,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 192
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 7.72,
+      "grad_norm": 0.5068610310554504,
+      "kl": 0.7425740361213684,
+      "learning_rate": 4.99352372834338e-06,
+      "loss": 0.0297,
+      "reward": 1.2116459608078003,
+      "reward_std": 3.281907558441162,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.6785714626312256,
+      "rewards/wrapped_driving_reward": -0.7169255018234253,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 193
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 7.76,
+      "grad_norm": 0.7546151876449585,
+      "kl": 0.5269922614097595,
+      "learning_rate": 4.993125462748714e-06,
+      "loss": 0.0211,
+      "reward": 2.08638334274292,
+      "reward_std": 0.4966030418872833,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.08638344705104828,
+      "rewards/wrapped_format_reward": 0.0,
+      "step": 194
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 7.8,
+      "grad_norm": 0.6085935831069946,
+      "kl": 0.7513608932495117,
+      "learning_rate": 4.992715330761167e-06,
+      "loss": 0.0301,
+      "reward": 0.6888164281845093,
+      "reward_std": 2.792956829071045,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.936183512210846,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 195
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 7.84,
+      "grad_norm": 0.6035264730453491,
+      "kl": 0.49863916635513306,
+      "learning_rate": 4.992293334332821e-06,
+      "loss": 0.0199,
+      "reward": 2.2989866733551025,
+      "reward_std": 0.7346133589744568,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.875,
+      "rewards/wrapped_driving_reward": -0.3260132670402527,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 196
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 7.88,
+      "grad_norm": 0.6059936285018921,
+      "kl": 0.5347932577133179,
+      "learning_rate": 4.9918594754722286e-06,
+      "loss": 0.0214,
+      "reward": 0.0036406517028808594,
+      "reward_std": 3.585599660873413,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.9963593482971191,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 197
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 7.92,
+      "grad_norm": 0.6184878945350647,
+      "kl": 0.45021745562553406,
+      "learning_rate": 4.991413756244404e-06,
+      "loss": 0.018,
+      "reward": 3.160332679748535,
+      "reward_std": 0.3758719265460968,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.5353326201438904,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 198
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 7.96,
+      "grad_norm": 0.5479530096054077,
+      "kl": 0.5874747633934021,
+      "learning_rate": 4.990956178770814e-06,
+      "loss": 0.0235,
+      "reward": 2.7277915477752686,
+      "reward_std": 0.21160702407360077,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.22779148817062378,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 199
+    },
+    {
+      "completion_length": 552.0,
+      "epoch": 8.0,
+      "grad_norm": 0.8959174156188965,
+      "kl": 0.7743228077888489,
+      "learning_rate": 4.990486745229364e-06,
+      "loss": 0.031,
+      "reward": -1.75,
+      "reward_std": 0.5692750215530396,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.875,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 200
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 8.04,
+      "grad_norm": 0.5334154367446899,
+      "kl": 0.5155819654464722,
+      "learning_rate": 4.990005457854392e-06,
+      "loss": 0.0206,
+      "reward": 2.358083963394165,
+      "reward_std": 0.6079920530319214,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9166666865348816,
+      "rewards/wrapped_driving_reward": -0.05858280509710312,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 201
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 8.08,
+      "grad_norm": 0.8115833401679993,
+      "kl": 0.7826488614082336,
+      "learning_rate": 4.989512318936654e-06,
+      "loss": 0.0313,
+      "reward": 2.9173591136932373,
+      "reward_std": 0.7723499536514282,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9772727489471436,
+      "rewards/wrapped_driving_reward": 0.31508636474609375,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 202
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 8.12,
+      "grad_norm": 1.8089442253112793,
+      "kl": 0.6300475597381592,
+      "learning_rate": 4.989007330823319e-06,
+      "loss": 0.0252,
+      "reward": 2.464756488800049,
+      "reward_std": 0.43305322527885437,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.2147563397884369,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 203
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 8.16,
+      "grad_norm": 19.12211799621582,
+      "kl": 5.240139484405518,
+      "learning_rate": 4.988490495917948e-06,
+      "loss": 0.2096,
+      "reward": -1.25,
+      "reward_std": 0.28867512941360474,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 204
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 8.2,
+      "grad_norm": 0.7113471627235413,
+      "kl": 0.6434310674667358,
+      "learning_rate": 4.987961816680493e-06,
+      "loss": 0.0257,
+      "reward": 0.7629947662353516,
+      "reward_std": 3.2651803493499756,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.237005352973938,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 205
+    },
+    {
+      "completion_length": 726.0,
+      "epoch": 8.24,
+      "grad_norm": 0.5587561726570129,
+      "kl": 0.759925365447998,
+      "learning_rate": 4.987421295627279e-06,
+      "loss": 0.0304,
+      "reward": 1.319366455078125,
+      "reward_std": 2.2250618934631348,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.8056334257125854,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 206
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 8.28,
+      "grad_norm": 0.5961654186248779,
+      "kl": 0.5305419564247131,
+      "learning_rate": 4.986868935330998e-06,
+      "loss": 0.0212,
+      "reward": 2.6727981567382812,
+      "reward_std": 0.3446647524833679,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.047798238694667816,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 207
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 8.32,
+      "grad_norm": 0.7032870650291443,
+      "kl": 0.6918764114379883,
+      "learning_rate": 4.986304738420684e-06,
+      "loss": 0.0277,
+      "reward": -0.13961869478225708,
+      "reward_std": 3.194305896759033,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.6937500238418579,
+      "rewards/wrapped_driving_reward": -2.0833687782287598,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 208
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 8.36,
+      "grad_norm": 1.351664662361145,
+      "kl": 0.44911444187164307,
+      "learning_rate": 4.985728707581717e-06,
+      "loss": 0.018,
+      "reward": 2.386528968811035,
+      "reward_std": 0.66311115026474,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8333333730697632,
+      "rewards/wrapped_driving_reward": 0.178195521235466,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 209
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 8.4,
+      "grad_norm": 0.8452834486961365,
+      "kl": 0.8271775841712952,
+      "learning_rate": 4.985140845555799e-06,
+      "loss": 0.0331,
+      "reward": 1.1259584426879883,
+      "reward_std": 2.119089365005493,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.8740414381027222,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 210
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 8.44,
+      "grad_norm": 1.1207107305526733,
+      "kl": 0.8840889930725098,
+      "learning_rate": 4.984541155140945e-06,
+      "loss": 0.0354,
+      "reward": 2.5146608352661133,
+      "reward_std": 0.5953065156936646,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.949999988079071,
+      "rewards/wrapped_driving_reward": 0.31466078758239746,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 211
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 8.48,
+      "grad_norm": 0.7914042472839355,
+      "kl": 0.9506034851074219,
+      "learning_rate": 4.9839296391914696e-06,
+      "loss": 0.038,
+      "reward": 3.054720401763916,
+      "reward_std": 0.5219823122024536,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.1797204613685608,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 212
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 8.52,
+      "grad_norm": 0.7439658045768738,
+      "kl": 0.5630563497543335,
+      "learning_rate": 4.98330630061797e-06,
+      "loss": 0.0225,
+      "reward": -0.11316037178039551,
+      "reward_std": 3.441740036010742,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.7381603717803955,
+      "rewards/wrapped_format_reward": 0.125,
+      "step": 213
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 8.56,
+      "grad_norm": 1.1477609872817993,
+      "kl": 1.0046734809875488,
+      "learning_rate": 4.982671142387316e-06,
+      "loss": 0.0402,
+      "reward": 3.0662035942077637,
+      "reward_std": 0.0674816370010376,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.5662035346031189,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 214
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 8.6,
+      "grad_norm": 0.46614599227905273,
+      "kl": 0.7210499048233032,
+      "learning_rate": 4.982024167522638e-06,
+      "loss": 0.0288,
+      "reward": 2.4149999618530273,
+      "reward_std": 0.46334606409072876,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.04000008851289749,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 215
+    },
+    {
+      "completion_length": 636.0,
+      "epoch": 8.64,
+      "grad_norm": 0.550269603729248,
+      "kl": 0.9145827293395996,
+      "learning_rate": 4.981365379103306e-06,
+      "loss": 0.0366,
+      "reward": 3.2154905796051025,
+      "reward_std": 0.5181944370269775,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.7154906988143921,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 216
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 8.68,
+      "grad_norm": 0.44090768694877625,
+      "kl": 0.9645712971687317,
+      "learning_rate": 4.980694780264918e-06,
+      "loss": 0.0386,
+      "reward": -1.274999976158142,
+      "reward_std": 0.4856266975402832,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9750000238418579,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 217
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 8.72,
+      "grad_norm": 1.1925650835037231,
+      "kl": 0.8605116605758667,
+      "learning_rate": 4.980012374199288e-06,
+      "loss": 0.0344,
+      "reward": 2.6870789527893066,
+      "reward_std": 0.7157343626022339,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.0629209354519844,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 218
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 8.76,
+      "grad_norm": 0.6395624876022339,
+      "kl": 0.8992162942886353,
+      "learning_rate": 4.979318164154426e-06,
+      "loss": 0.036,
+      "reward": -1.28125,
+      "reward_std": 0.32874444127082825,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.96875,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 219
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 8.8,
+      "grad_norm": 0.5243760943412781,
+      "kl": 0.7105860710144043,
+      "learning_rate": 4.978612153434527e-06,
+      "loss": 0.0284,
+      "reward": 3.133758783340454,
+      "reward_std": 0.4695283770561218,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.25875866413116455,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 220
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 8.84,
+      "grad_norm": 0.5799823999404907,
+      "kl": 0.8737132549285889,
+      "learning_rate": 4.97789434539995e-06,
+      "loss": 0.0349,
+      "reward": 1.885632038116455,
+      "reward_std": 1.8393102884292603,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.8643680214881897,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 221
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 8.88,
+      "grad_norm": 0.46507924795150757,
+      "kl": 0.9933305382728577,
+      "learning_rate": 4.977164743467206e-06,
+      "loss": 0.0397,
+      "reward": -2.049999952316284,
+      "reward_std": 0.8225975036621094,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.699999988079071,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 222
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 8.92,
+      "grad_norm": 0.557388961315155,
+      "kl": 0.6470035314559937,
+      "learning_rate": 4.976423351108943e-06,
+      "loss": 0.0259,
+      "reward": -1.25,
+      "reward_std": 0.28867512941360474,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 223
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 8.96,
+      "grad_norm": 0.7340157628059387,
+      "kl": 0.6879977583885193,
+      "learning_rate": 4.975670171853926e-06,
+      "loss": 0.0275,
+      "reward": 2.4769201278686523,
+      "reward_std": 0.25078660249710083,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.875,
+      "rewards/wrapped_driving_reward": 0.10192020237445831,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 224
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 9.0,
+      "grad_norm": 1.4428342580795288,
+      "kl": 1.1961815357208252,
+      "learning_rate": 4.97490520928702e-06,
+      "loss": 0.0478,
+      "reward": 3.0022120475769043,
+      "reward_std": 0.3715779185295105,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.6272119879722595,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 225
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 9.04,
+      "grad_norm": 0.5274134874343872,
+      "kl": 0.8709424734115601,
+      "learning_rate": 4.974128467049177e-06,
+      "loss": 0.0348,
+      "reward": 1.142529845237732,
+      "reward_std": 3.107614040374756,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9824702143669128,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 226
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 9.08,
+      "grad_norm": 0.7395073771476746,
+      "kl": 0.2381790727376938,
+      "learning_rate": 4.9733399488374115e-06,
+      "loss": 0.0095,
+      "reward": 3.141404628753662,
+      "reward_std": 0.23124907910823822,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.14140476286411285,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 227
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 9.12,
+      "grad_norm": 0.5715224742889404,
+      "kl": 0.47389835119247437,
+      "learning_rate": 4.972539658404793e-06,
+      "loss": 0.019,
+      "reward": 2.177046775817871,
+      "reward_std": 0.8025394678115845,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.875,
+      "rewards/wrapped_driving_reward": -0.32295334339141846,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 228
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 9.16,
+      "grad_norm": 0.614587128162384,
+      "kl": 0.877197802066803,
+      "learning_rate": 4.971727599560418e-06,
+      "loss": 0.0351,
+      "reward": 2.604313373565674,
+      "reward_std": 0.43783459067344666,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.020686477422714233,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 229
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 9.2,
+      "grad_norm": 0.49547889828681946,
+      "kl": 0.8945198655128479,
+      "learning_rate": 4.970903776169403e-06,
+      "loss": 0.0358,
+      "reward": 2.9887380599975586,
+      "reward_std": 0.49843111634254456,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9750000238418579,
+      "rewards/wrapped_driving_reward": 0.3887380361557007,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 230
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 9.24,
+      "grad_norm": 0.6391835808753967,
+      "kl": 0.8877568244934082,
+      "learning_rate": 4.9700681921528495e-06,
+      "loss": 0.0355,
+      "reward": 2.256884813308716,
+      "reward_std": 0.810869038105011,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9285714626312256,
+      "rewards/wrapped_driving_reward": -0.04668661952018738,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 231
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 9.28,
+      "grad_norm": 0.7567249536514282,
+      "kl": 0.9957519769668579,
+      "learning_rate": 4.9692208514878445e-06,
+      "loss": 0.0398,
+      "reward": 2.5888562202453613,
+      "reward_std": 0.6119778156280518,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9565972089767456,
+      "rewards/wrapped_driving_reward": 0.13225889205932617,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 232
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 9.32,
+      "grad_norm": 0.63723224401474,
+      "kl": 0.8475660681724548,
+      "learning_rate": 4.968361758207428e-06,
+      "loss": 0.0339,
+      "reward": -1.8181817531585693,
+      "reward_std": 0.23764224350452423,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9318181872367859,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 233
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 9.36,
+      "grad_norm": 0.48718228936195374,
+      "kl": 1.1619231700897217,
+      "learning_rate": 4.9674909164005805e-06,
+      "loss": 0.0465,
+      "reward": 3.505831718444824,
+      "reward_std": 0.3402004837989807,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.6308315992355347,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 234
+    },
+    {
+      "completion_length": 676.0,
+      "epoch": 9.4,
+      "grad_norm": 0.8502306342124939,
+      "kl": 1.0721936225891113,
+      "learning_rate": 4.966608330212198e-06,
+      "loss": 0.0429,
+      "reward": 3.320247173309326,
+      "reward_std": 0.5307442545890808,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.5702470541000366,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 235
+    },
+    {
+      "completion_length": 635.0,
+      "epoch": 9.44,
+      "grad_norm": 0.6630327105522156,
+      "kl": 0.8887981176376343,
+      "learning_rate": 4.965714003843079e-06,
+      "loss": 0.0356,
+      "reward": 2.481696844100952,
+      "reward_std": 0.16960932314395905,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.018303271383047104,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 236
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 9.48,
+      "grad_norm": 0.45822253823280334,
+      "kl": 0.851169764995575,
+      "learning_rate": 4.9648079415499e-06,
+      "loss": 0.034,
+      "reward": -1.8214285373687744,
+      "reward_std": 0.5639389753341675,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9285714626312256,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 237
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 9.52,
+      "grad_norm": 0.41027647256851196,
+      "kl": 1.0325894355773926,
+      "learning_rate": 4.963890147645195e-06,
+      "loss": 0.0413,
+      "reward": 2.844029426574707,
+      "reward_std": 0.30431851744651794,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.030970722436904907,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 238
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 9.56,
+      "grad_norm": 3.95577073097229,
+      "kl": 1.8168144226074219,
+      "learning_rate": 4.962960626497339e-06,
+      "loss": 0.0727,
+      "reward": 2.185295343399048,
+      "reward_std": 0.4056129455566406,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.31470465660095215,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 239
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 9.6,
+      "grad_norm": 0.84366774559021,
+      "kl": 1.09842050075531,
+      "learning_rate": 4.962019382530521e-06,
+      "loss": 0.0439,
+      "reward": 3.1190567016601562,
+      "reward_std": 0.16080652177333832,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.1190568059682846,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 240
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 9.64,
+      "grad_norm": 0.40419045090675354,
+      "kl": 1.4857383966445923,
+      "learning_rate": 4.961066420224729e-06,
+      "loss": 0.0594,
+      "reward": 2.7421183586120605,
+      "reward_std": 0.3939764201641083,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.007881544530391693,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 241
+    },
+    {
+      "completion_length": 703.0,
+      "epoch": 9.68,
+      "grad_norm": 0.5581960678100586,
+      "kl": 0.8319287896156311,
+      "learning_rate": 4.960101744115727e-06,
+      "loss": 0.0333,
+      "reward": 1.3577722311019897,
+      "reward_std": 0.8074946999549866,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.8922277688980103,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 242
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 9.72,
+      "grad_norm": 1.4876646995544434,
+      "kl": 1.1022382974624634,
+      "learning_rate": 4.959125358795031e-06,
+      "loss": 0.0441,
+      "reward": -2.125,
+      "reward_std": 1.314977765083313,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 243
+    },
+    {
+      "completion_length": 733.0,
+      "epoch": 9.76,
+      "grad_norm": 0.7890307903289795,
+      "kl": 1.5635696649551392,
+      "learning_rate": 4.958137268909887e-06,
+      "loss": 0.0625,
+      "reward": 1.0345841646194458,
+      "reward_std": 2.7603647708892822,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.3404158353805542,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 244
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 9.8,
+      "grad_norm": 0.44861987233161926,
+      "kl": 0.25233525037765503,
+      "learning_rate": 4.957137479163253e-06,
+      "loss": 0.0101,
+      "reward": -1.7541667222976685,
+      "reward_std": 0.3909568190574646,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8708333373069763,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 245
+    },
+    {
+      "completion_length": 533.0,
+      "epoch": 9.84,
+      "grad_norm": 0.5020561218261719,
+      "kl": 0.9620947241783142,
+      "learning_rate": 4.956125994313775e-06,
+      "loss": 0.0385,
+      "reward": 3.3699028491973877,
+      "reward_std": 0.5193167924880981,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.4949028491973877,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 246
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 9.88,
+      "grad_norm": 0.7062340974807739,
+      "kl": 0.9898033738136292,
+      "learning_rate": 4.95510281917576e-06,
+      "loss": 0.0396,
+      "reward": -1.875,
+      "reward_std": 1.1814539432525635,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 247
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 9.92,
+      "grad_norm": 0.44811582565307617,
+      "kl": 0.43252551555633545,
+      "learning_rate": 4.9540679586191605e-06,
+      "loss": 0.0173,
+      "reward": 2.317924976348877,
+      "reward_std": 0.17351354658603668,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.06792493164539337,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 248
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 9.96,
+      "grad_norm": 0.47783583402633667,
+      "kl": 0.9962712526321411,
+      "learning_rate": 4.953021417569545e-06,
+      "loss": 0.0399,
+      "reward": 3.022937059402466,
+      "reward_std": 0.4499557316303253,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.14793699979782104,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 249
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 10.0,
+      "grad_norm": 0.5202720761299133,
+      "kl": 0.5016875863075256,
+      "learning_rate": 4.9519632010080765e-06,
+      "loss": 0.0201,
+      "reward": 1.3368468284606934,
+      "reward_std": 3.5631000995635986,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.6631531715393066,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 250
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 10.04,
+      "grad_norm": 0.889390766620636,
+      "kl": 1.2343968152999878,
+      "learning_rate": 4.950893313971492e-06,
+      "loss": 0.0494,
+      "reward": 3.506786346435547,
+      "reward_std": 0.3962436020374298,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9444444179534912,
+      "rewards/wrapped_driving_reward": 0.8123420476913452,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 251
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 10.08,
+      "grad_norm": 0.5827829241752625,
+      "kl": 0.948403000831604,
+      "learning_rate": 4.949811761552074e-06,
+      "loss": 0.0379,
+      "reward": 2.5721993446350098,
+      "reward_std": 0.5560285449028015,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9583333134651184,
+      "rewards/wrapped_driving_reward": -0.011133967898786068,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 252
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 10.12,
+      "grad_norm": 0.5650044679641724,
+      "kl": 0.9299434423446655,
+      "learning_rate": 4.9487185488976284e-06,
+      "loss": 0.0372,
+      "reward": -1.716269850730896,
+      "reward_std": 0.5084477663040161,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.908730149269104,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 253
+    },
+    {
+      "completion_length": 599.0,
+      "epoch": 10.16,
+      "grad_norm": 0.4880934953689575,
+      "kl": 0.7951986789703369,
+      "learning_rate": 4.94761368121146e-06,
+      "loss": 0.0318,
+      "reward": 2.573094367980957,
+      "reward_std": 0.27557268738746643,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.1769055724143982,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 254
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 10.2,
+      "grad_norm": 0.8892874121665955,
+      "kl": 0.7362837195396423,
+      "learning_rate": 4.9464971637523465e-06,
+      "loss": 0.0295,
+      "reward": -1.28125,
+      "reward_std": 0.4827762544155121,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.96875,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 255
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 10.24,
+      "grad_norm": 0.8154737949371338,
+      "kl": 0.9433515667915344,
+      "learning_rate": 4.9453690018345144e-06,
+      "loss": 0.0377,
+      "reward": 1.883481502532959,
+      "reward_std": 0.9224264025688171,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.4915185868740082,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 256
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 10.28,
+      "grad_norm": 0.587221086025238,
+      "kl": 0.7820435166358948,
+      "learning_rate": 4.944229200827616e-06,
+      "loss": 0.0313,
+      "reward": -1.1607142686843872,
+      "reward_std": 0.23600271344184875,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9642857313156128,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 257
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 10.32,
+      "grad_norm": 0.7322145700454712,
+      "kl": 0.9088730812072754,
+      "learning_rate": 4.943077766156698e-06,
+      "loss": 0.0364,
+      "reward": 0.9441255927085876,
+      "reward_std": 1.5783616304397583,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9791666865348816,
+      "rewards/wrapped_driving_reward": -1.910041093826294,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 258
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 10.36,
+      "grad_norm": 0.7966383099555969,
+      "kl": 1.125408411026001,
+      "learning_rate": 4.941914703302181e-06,
+      "loss": 0.045,
+      "reward": 2.580202102661133,
+      "reward_std": 0.40770646929740906,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.08020199090242386,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 259
+    },
+    {
+      "completion_length": 695.0,
+      "epoch": 10.4,
+      "grad_norm": 0.4875122308731079,
+      "kl": 0.8961836695671082,
+      "learning_rate": 4.9407400177998335e-06,
+      "loss": 0.0358,
+      "reward": 2.2389979362487793,
+      "reward_std": 0.7594300508499146,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.7610019445419312,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 260
+    },
+    {
+      "completion_length": 612.0,
+      "epoch": 10.44,
+      "grad_norm": 0.8443101048469543,
+      "kl": 0.8345216512680054,
+      "learning_rate": 4.939553715240741e-06,
+      "loss": 0.0334,
+      "reward": 2.9486937522888184,
+      "reward_std": 0.7755388617515564,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.05130642652511597,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 261
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 10.48,
+      "grad_norm": 0.4315735697746277,
+      "kl": 0.5944791436195374,
+      "learning_rate": 4.938355801271282e-06,
+      "loss": 0.0238,
+      "reward": -0.26047587394714355,
+      "reward_std": 3.4582109451293945,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.6354758739471436,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 262
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 10.52,
+      "grad_norm": 0.4449390172958374,
+      "kl": 1.0638983249664307,
+      "learning_rate": 4.937146281593103e-06,
+      "loss": 0.0426,
+      "reward": 3.349001407623291,
+      "reward_std": 0.18792293965816498,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.7240012884140015,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 263
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 10.56,
+      "grad_norm": 0.5087334513664246,
+      "kl": 0.9471940994262695,
+      "learning_rate": 4.935925161963089e-06,
+      "loss": 0.0379,
+      "reward": -1.625,
+      "reward_std": 1.25,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 264
+    },
+    {
+      "completion_length": 732.0,
+      "epoch": 10.6,
+      "grad_norm": 0.5004269480705261,
+      "kl": 0.9943680167198181,
+      "learning_rate": 4.9346924481933345e-06,
+      "loss": 0.0398,
+      "reward": 3.4356508255004883,
+      "reward_std": 0.5672562122344971,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9583333134651184,
+      "rewards/wrapped_driving_reward": 0.7273174524307251,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 265
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 10.64,
+      "grad_norm": 0.39916032552719116,
+      "kl": 1.0476347208023071,
+      "learning_rate": 4.933448146151122e-06,
+      "loss": 0.0419,
+      "reward": 2.414046049118042,
+      "reward_std": 0.3546769917011261,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8392857313156128,
+      "rewards/wrapped_driving_reward": 0.07476034015417099,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 266
+    },
+    {
+      "completion_length": 735.0,
+      "epoch": 10.68,
+      "grad_norm": 0.4085545241832733,
+      "kl": 0.9289141297340393,
+      "learning_rate": 4.932192261758885e-06,
+      "loss": 0.0372,
+      "reward": -1.5,
+      "reward_std": 0.5773502588272095,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 267
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 10.72,
+      "grad_norm": 0.7282282114028931,
+      "kl": 0.5237314701080322,
+      "learning_rate": 4.930924800994192e-06,
+      "loss": 0.0209,
+      "reward": 1.163808822631836,
+      "reward_std": 3.132412910461426,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9611911177635193,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 268
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 10.76,
+      "grad_norm": 0.5592875480651855,
+      "kl": 1.2230463027954102,
+      "learning_rate": 4.929645769889704e-06,
+      "loss": 0.0489,
+      "reward": 1.8117026090621948,
+      "reward_std": 1.2372390031814575,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.9382973909378052,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 269
+    },
+    {
+      "completion_length": 666.0,
+      "epoch": 10.8,
+      "grad_norm": 0.6668244004249573,
+      "kl": 1.1006290912628174,
+      "learning_rate": 4.928355174533153e-06,
+      "loss": 0.044,
+      "reward": 0.7610301971435547,
+      "reward_std": 1.6584932804107666,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -2.113969564437866,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 270
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 10.84,
+      "grad_norm": 0.9733495712280273,
+      "kl": 0.7359632253646851,
+      "learning_rate": 4.927053021067321e-06,
+      "loss": 0.0294,
+      "reward": 2.9857444763183594,
+      "reward_std": 0.5656386613845825,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.11074452847242355,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 271
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 10.88,
+      "grad_norm": 0.9902675747871399,
+      "kl": 0.6186420321464539,
+      "learning_rate": 4.925739315689991e-06,
+      "loss": 0.0247,
+      "reward": 2.7555360794067383,
+      "reward_std": 0.022727251052856445,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9886363744735718,
+      "rewards/wrapped_driving_reward": 0.7668997645378113,
+      "rewards/wrapped_format_reward": 0.0,
+      "step": 272
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 10.92,
+      "grad_norm": 0.5004103183746338,
+      "kl": 1.1375739574432373,
+      "learning_rate": 4.924414064653938e-06,
+      "loss": 0.0455,
+      "reward": 2.6497280597686768,
+      "reward_std": 0.5490097999572754,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.14972800016403198,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 273
+    },
+    {
+      "completion_length": 615.0,
+      "epoch": 10.96,
+      "grad_norm": 0.686726450920105,
+      "kl": 0.96458899974823,
+      "learning_rate": 4.923077274266886e-06,
+      "loss": 0.0386,
+      "reward": 2.8679394721984863,
+      "reward_std": 0.5836524367332458,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9027777910232544,
+      "rewards/wrapped_driving_reward": 0.2151617854833603,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 274
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 11.0,
+      "grad_norm": 0.41074368357658386,
+      "kl": 0.7832292914390564,
+      "learning_rate": 4.9217289508914836e-06,
+      "loss": 0.0313,
+      "reward": 2.4006309509277344,
+      "reward_std": 0.9599378108978271,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.34936898946762085,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 275
+    },
+    {
+      "completion_length": 417.0,
+      "epoch": 11.04,
+      "grad_norm": 0.6297289133071899,
+      "kl": 0.5816258788108826,
+      "learning_rate": 4.92036910094527e-06,
+      "loss": 0.0233,
+      "reward": 2.59334397315979,
+      "reward_std": 0.5557723045349121,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.09334398806095123,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 276
+    },
+    {
+      "completion_length": 534.0,
+      "epoch": 11.08,
+      "grad_norm": 0.5123348832130432,
+      "kl": 0.8924129605293274,
+      "learning_rate": 4.91899773090065e-06,
+      "loss": 0.0357,
+      "reward": 1.080291986465454,
+      "reward_std": 1.076037883758545,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9861111044883728,
+      "rewards/wrapped_driving_reward": -1.7808191776275635,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 277
+    },
+    {
+      "completion_length": 599.0,
+      "epoch": 11.12,
+      "grad_norm": 0.46222391724586487,
+      "kl": 0.4757728576660156,
+      "learning_rate": 4.917614847284858e-06,
+      "loss": 0.019,
+      "reward": 2.672243118286133,
+      "reward_std": 0.4222791790962219,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9791666865348816,
+      "rewards/wrapped_driving_reward": -0.3069234788417816,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 278
+    },
+    {
+      "completion_length": 749.0,
+      "epoch": 11.16,
+      "grad_norm": 0.586867094039917,
+      "kl": 1.4221863746643066,
+      "learning_rate": 4.91622045667993e-06,
+      "loss": 0.0569,
+      "reward": 2.0654354095458984,
+      "reward_std": 2.719116687774658,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.6845643520355225,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 279
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 11.2,
+      "grad_norm": 0.4978845417499542,
+      "kl": 0.7794169783592224,
+      "learning_rate": 4.914814565722671e-06,
+      "loss": 0.0312,
+      "reward": 3.286668300628662,
+      "reward_std": 0.5568961501121521,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.5366683602333069,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 280
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 11.24,
+      "grad_norm": 1.042169213294983,
+      "kl": 1.359074592590332,
+      "learning_rate": 4.913397181104623e-06,
+      "loss": 0.0544,
+      "reward": 0.6235643029212952,
+      "reward_std": 2.8482930660247803,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.6264357566833496,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 281
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 11.28,
+      "grad_norm": 0.6336880922317505,
+      "kl": 1.260665774345398,
+      "learning_rate": 4.9119683095720325e-06,
+      "loss": 0.0504,
+      "reward": 3.2773139476776123,
+      "reward_std": 0.8147690892219543,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": 0.6523139476776123,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 282
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 11.32,
+      "grad_norm": 0.5326210260391235,
+      "kl": 1.2928297519683838,
+      "learning_rate": 4.9105279579258234e-06,
+      "loss": 0.0517,
+      "reward": 3.1767990589141846,
+      "reward_std": 0.6041759252548218,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.4267989993095398,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 283
+    },
+    {
+      "completion_length": 520.0,
+      "epoch": 11.36,
+      "grad_norm": 0.48858872056007385,
+      "kl": 0.4190160632133484,
+      "learning_rate": 4.909076133021558e-06,
+      "loss": 0.0168,
+      "reward": -1.4375,
+      "reward_std": 0.5907269716262817,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9375,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 284
+    },
+    {
+      "completion_length": 676.0,
+      "epoch": 11.4,
+      "grad_norm": 0.43427374958992004,
+      "kl": 1.0146484375,
+      "learning_rate": 4.907612841769407e-06,
+      "loss": 0.0406,
+      "reward": -1.25,
+      "reward_std": 0.5,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 285
+    },
+    {
+      "completion_length": 615.0,
+      "epoch": 11.44,
+      "grad_norm": 0.5486767888069153,
+      "kl": 0.7701943516731262,
+      "learning_rate": 4.906138091134118e-06,
+      "loss": 0.0308,
+      "reward": 3.0628390312194824,
+      "reward_std": 0.11028631031513214,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.06283894926309586,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 286
+    },
+    {
+      "completion_length": 595.0,
+      "epoch": 11.48,
+      "grad_norm": 0.4506373107433319,
+      "kl": 0.901443600654602,
+      "learning_rate": 4.904651888134982e-06,
+      "loss": 0.0361,
+      "reward": -1.25,
+      "reward_std": 0.5,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 287
+    },
+    {
+      "completion_length": 744.0,
+      "epoch": 11.52,
+      "grad_norm": 0.4597472846508026,
+      "kl": 1.1629990339279175,
+      "learning_rate": 4.903154239845798e-06,
+      "loss": 0.0465,
+      "reward": 2.7203586101531982,
+      "reward_std": 0.7249525785446167,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.84375,
+      "rewards/wrapped_driving_reward": 0.0016086697578430176,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 288
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 11.56,
+      "grad_norm": 0.5143890976905823,
+      "kl": 0.9513099789619446,
+      "learning_rate": 4.901645153394838e-06,
+      "loss": 0.0381,
+      "reward": -1.25,
+      "reward_std": 0.28867512941360474,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 289
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 11.6,
+      "grad_norm": 0.532035768032074,
+      "kl": 0.6954802870750427,
+      "learning_rate": 4.900124635964823e-06,
+      "loss": 0.0278,
+      "reward": 3.240325689315796,
+      "reward_std": 0.25314152240753174,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9722222089767456,
+      "rewards/wrapped_driving_reward": 0.2681034803390503,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 290
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 11.64,
+      "grad_norm": 0.7568380832672119,
+      "kl": 0.890608549118042,
+      "learning_rate": 4.898592694792871e-06,
+      "loss": 0.0356,
+      "reward": 3.097019672393799,
+      "reward_std": 0.5597526431083679,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.22201967239379883,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 291
+    },
+    {
+      "completion_length": 597.0,
+      "epoch": 11.68,
+      "grad_norm": 0.5061165690422058,
+      "kl": 0.8536003232002258,
+      "learning_rate": 4.897049337170483e-06,
+      "loss": 0.0341,
+      "reward": 2.722294330596924,
+      "reward_std": 0.21757638454437256,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9444444179534912,
+      "rewards/wrapped_driving_reward": 0.15284982323646545,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 292
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 11.72,
+      "grad_norm": 0.5000802278518677,
+      "kl": 0.9599359035491943,
+      "learning_rate": 4.895494570443492e-06,
+      "loss": 0.0384,
+      "reward": 2.7536168098449707,
+      "reward_std": 0.6582252383232117,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.12138298898935318,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 293
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 11.76,
+      "grad_norm": 0.5710813999176025,
+      "kl": 0.9540033340454102,
+      "learning_rate": 4.8939284020120365e-06,
+      "loss": 0.0382,
+      "reward": 2.571502208709717,
+      "reward_std": 0.4067968428134918,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9166666865348816,
+      "rewards/wrapped_driving_reward": 0.029835540801286697,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 294
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 11.8,
+      "grad_norm": 1.1970958709716797,
+      "kl": 1.2357957363128662,
+      "learning_rate": 4.8923508393305224e-06,
+      "loss": 0.0494,
+      "reward": 0.6461101770401001,
+      "reward_std": 2.581754207611084,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.8538898229599,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 295
+    },
+    {
+      "completion_length": 738.0,
+      "epoch": 11.84,
+      "grad_norm": 0.3895174264907837,
+      "kl": 0.5224874019622803,
+      "learning_rate": 4.890761889907589e-06,
+      "loss": 0.0209,
+      "reward": 2.2970166206359863,
+      "reward_std": 0.6513614058494568,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9583333134651184,
+      "rewards/wrapped_driving_reward": -0.16131654381752014,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 296
+    },
+    {
+      "completion_length": 581.0,
+      "epoch": 11.88,
+      "grad_norm": 0.5378090739250183,
+      "kl": 0.9656004309654236,
+      "learning_rate": 4.8891615613060715e-06,
+      "loss": 0.0386,
+      "reward": 2.449643135070801,
+      "reward_std": 1.1916462182998657,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.5503568649291992,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 297
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 11.92,
+      "grad_norm": 1.0380306243896484,
+      "kl": 1.3637113571166992,
+      "learning_rate": 4.887549861142967e-06,
+      "loss": 0.0545,
+      "reward": -1.125,
+      "reward_std": 0.25,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 298
+    },
+    {
+      "completion_length": 660.0,
+      "epoch": 11.96,
+      "grad_norm": 0.5935966968536377,
+      "kl": 1.596313238143921,
+      "learning_rate": 4.885926797089396e-06,
+      "loss": 0.0639,
+      "reward": 1.3272292613983154,
+      "reward_std": 2.9181787967681885,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9227706789970398,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 299
+    },
+    {
+      "completion_length": 605.0,
+      "epoch": 12.0,
+      "grad_norm": 0.5052400231361389,
+      "kl": 1.1359528303146362,
+      "learning_rate": 4.884292376870567e-06,
+      "loss": 0.0454,
+      "reward": 1.1219098567962646,
+      "reward_std": 2.8648571968078613,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.3780901432037354,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 300
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 12.04,
+      "grad_norm": 0.42255502939224243,
+      "kl": 1.3561640977859497,
+      "learning_rate": 4.882646608265743e-06,
+      "loss": 0.0542,
+      "reward": -1.375,
+      "reward_std": 0.4787135720252991,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 301
+    },
+    {
+      "completion_length": 672.0,
+      "epoch": 12.08,
+      "grad_norm": 0.7515414953231812,
+      "kl": 1.0097947120666504,
+      "learning_rate": 4.880989499108196e-06,
+      "loss": 0.0404,
+      "reward": 2.7926979064941406,
+      "reward_std": 0.243763267993927,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.4176979064941406,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 302
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 12.12,
+      "grad_norm": 0.482781320810318,
+      "kl": 1.0105189085006714,
+      "learning_rate": 4.8793210572851795e-06,
+      "loss": 0.0404,
+      "reward": 1.3847792148590088,
+      "reward_std": 1.7117525339126587,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.4902207851409912,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 303
+    },
+    {
+      "completion_length": 561.0,
+      "epoch": 12.16,
+      "grad_norm": 2.1605520248413086,
+      "kl": 0.9172693490982056,
+      "learning_rate": 4.8776412907378845e-06,
+      "loss": 0.0367,
+      "reward": 2.826827049255371,
+      "reward_std": 0.5229109525680542,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9772727489471436,
+      "rewards/wrapped_driving_reward": -0.025445779785513878,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 304
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 12.2,
+      "grad_norm": 0.7399263381958008,
+      "kl": 1.4161840677261353,
+      "learning_rate": 4.875950207461403e-06,
+      "loss": 0.0566,
+      "reward": 2.040844202041626,
+      "reward_std": 3.3612470626831055,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7321428656578064,
+      "rewards/wrapped_driving_reward": -0.44129857420921326,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 305
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 12.24,
+      "grad_norm": 0.5619301795959473,
+      "kl": 1.1429786682128906,
+      "learning_rate": 4.874247815504693e-06,
+      "loss": 0.0457,
+      "reward": 3.1347854137420654,
+      "reward_std": 0.2799624502658844,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9861111044883728,
+      "rewards/wrapped_driving_reward": 0.1486743986606598,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 306
+    },
+    {
+      "completion_length": 532.0,
+      "epoch": 12.28,
+      "grad_norm": 1.447466492652893,
+      "kl": 0.9073767066001892,
+      "learning_rate": 4.872534122970536e-06,
+      "loss": 0.0363,
+      "reward": -1.125,
+      "reward_std": 0.25,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 307
+    },
+    {
+      "completion_length": 397.0,
+      "epoch": 12.32,
+      "grad_norm": 3.610201597213745,
+      "kl": 0.4948464035987854,
+      "learning_rate": 4.870809138015499e-06,
+      "loss": 0.0198,
+      "reward": 1.3169913291931152,
+      "reward_std": 3.2238197326660156,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9330087304115295,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 308
+    },
+    {
+      "completion_length": 631.0,
+      "epoch": 12.36,
+      "grad_norm": 1.619842529296875,
+      "kl": 0.6367069482803345,
+      "learning_rate": 4.8690728688499e-06,
+      "loss": 0.0255,
+      "reward": -1.125,
+      "reward_std": 0.25,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 309
+    },
+    {
+      "completion_length": 597.0,
+      "epoch": 12.4,
+      "grad_norm": 1.2359195947647095,
+      "kl": 0.6074169278144836,
+      "learning_rate": 4.867325323737765e-06,
+      "loss": 0.0243,
+      "reward": 2.357518196105957,
+      "reward_std": 0.7375664710998535,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.6424819231033325,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 310
+    },
+    {
+      "completion_length": 469.0,
+      "epoch": 12.44,
+      "grad_norm": 4.1004719734191895,
+      "kl": 0.5925214290618896,
+      "learning_rate": 4.865566510996787e-06,
+      "loss": 0.0237,
+      "reward": 2.563359022140503,
+      "reward_std": 0.33094266057014465,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.96875,
+      "rewards/wrapped_driving_reward": -0.15539111196994781,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 311
+    },
+    {
+      "completion_length": 740.0,
+      "epoch": 12.48,
+      "grad_norm": 11.807194709777832,
+      "kl": 1.1617281436920166,
+      "learning_rate": 4.863796438998293e-06,
+      "loss": 0.0465,
+      "reward": 2.8380610942840576,
+      "reward_std": 0.5543819069862366,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.08806122839450836,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 312
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 12.52,
+      "grad_norm": 5.792469024658203,
+      "kl": 0.404826819896698,
+      "learning_rate": 4.862015116167195e-06,
+      "loss": 0.0162,
+      "reward": 2.552396774291992,
+      "reward_std": 0.6585280895233154,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.1976032257080078,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 313
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 12.56,
+      "grad_norm": 8.077740669250488,
+      "kl": 0.5904800295829773,
+      "learning_rate": 4.860222550981961e-06,
+      "loss": 0.0236,
+      "reward": 2.346454381942749,
+      "reward_std": 0.9884578585624695,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.403545618057251,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 314
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 12.6,
+      "grad_norm": 4.022387504577637,
+      "kl": 0.7494456768035889,
+      "learning_rate": 4.858418751974564e-06,
+      "loss": 0.03,
+      "reward": -1.1458332538604736,
+      "reward_std": 0.1717960387468338,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8541666865348816,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 315
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 12.64,
+      "grad_norm": 4.89057731628418,
+      "kl": 0.7992514967918396,
+      "learning_rate": 4.856603727730446e-06,
+      "loss": 0.032,
+      "reward": 2.9716575145721436,
+      "reward_std": 0.8948715329170227,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.949999988079071,
+      "rewards/wrapped_driving_reward": 0.39665764570236206,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 316
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 12.68,
+      "grad_norm": 4.50545072555542,
+      "kl": 0.706625759601593,
+      "learning_rate": 4.854777486888481e-06,
+      "loss": 0.0283,
+      "reward": -0.9577881693840027,
+      "reward_std": 2.518893003463745,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -3.0827882289886475,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 317
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 12.72,
+      "grad_norm": 1.048317790031433,
+      "kl": 0.4814412593841553,
+      "learning_rate": 4.852940038140927e-06,
+      "loss": 0.0193,
+      "reward": 2.9507060050964355,
+      "reward_std": 0.5745217204093933,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.32570600509643555,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 318
+    },
+    {
+      "completion_length": 561.0,
+      "epoch": 12.76,
+      "grad_norm": 2.460906744003296,
+      "kl": 0.2864713668823242,
+      "learning_rate": 4.8510913902333876e-06,
+      "loss": 0.0115,
+      "reward": 2.4077253341674805,
+      "reward_std": 0.4348691999912262,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9375,
+      "rewards/wrapped_driving_reward": -0.5297746658325195,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 319
+    },
+    {
+      "completion_length": 469.0,
+      "epoch": 12.8,
+      "grad_norm": 4.8929314613342285,
+      "kl": 0.5974794626235962,
+      "learning_rate": 4.849231551964771e-06,
+      "loss": 0.0239,
+      "reward": 2.981731653213501,
+      "reward_std": 0.12611062824726105,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9583333134651184,
+      "rewards/wrapped_driving_reward": 0.02339838445186615,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 320
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 12.84,
+      "grad_norm": 1.4380724430084229,
+      "kl": 0.7370598912239075,
+      "learning_rate": 4.8473605321872484e-06,
+      "loss": 0.0295,
+      "reward": 2.465222120285034,
+      "reward_std": 0.3577950894832611,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.034777797758579254,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 321
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 12.88,
+      "grad_norm": 0.7334043979644775,
+      "kl": 0.6930970549583435,
+      "learning_rate": 4.845478339806211e-06,
+      "loss": 0.0277,
+      "reward": 2.6659016609191895,
+      "reward_std": 0.4846745729446411,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.04090156406164169,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 322
+    },
+    {
+      "completion_length": 704.0,
+      "epoch": 12.92,
+      "grad_norm": 2.4858033657073975,
+      "kl": 0.7707926034927368,
+      "learning_rate": 4.843584983780225e-06,
+      "loss": 0.0308,
+      "reward": 1.1393108367919922,
+      "reward_std": 2.8294146060943604,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.625,
+      "rewards/wrapped_driving_reward": -1.2356891632080078,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 323
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 12.96,
+      "grad_norm": 2.020785093307495,
+      "kl": 1.2838598489761353,
+      "learning_rate": 4.841680473120994e-06,
+      "loss": 0.0514,
+      "reward": -1.375,
+      "reward_std": 0.4787135720252991,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 324
+    },
+    {
+      "completion_length": 739.0,
+      "epoch": 13.0,
+      "grad_norm": 0.6641396284103394,
+      "kl": 1.4216065406799316,
+      "learning_rate": 4.839764816893315e-06,
+      "loss": 0.0569,
+      "reward": 2.8785228729248047,
+      "reward_std": 0.4891079366207123,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.2535229027271271,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 325
+    },
+    {
+      "completion_length": 477.0,
+      "epoch": 13.04,
+      "grad_norm": 1.4598071575164795,
+      "kl": 0.6853589415550232,
+      "learning_rate": 4.83783802421503e-06,
+      "loss": 0.0274,
+      "reward": 1.5590646266937256,
+      "reward_std": 3.725144386291504,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.6909353733062744,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 326
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 13.08,
+      "grad_norm": 0.4631199240684509,
+      "kl": 1.2964632511138916,
+      "learning_rate": 4.835900104256989e-06,
+      "loss": 0.0519,
+      "reward": 0.21913164854049683,
+      "reward_std": 2.63443660736084,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -2.0308685302734375,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 327
+    },
+    {
+      "completion_length": 559.0,
+      "epoch": 13.12,
+      "grad_norm": 0.8331103324890137,
+      "kl": 0.8320713639259338,
+      "learning_rate": 4.833951066243004e-06,
+      "loss": 0.0333,
+      "reward": -1.850000023841858,
+      "reward_std": 1.12101149559021,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.6499999761581421,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 328
+    },
+    {
+      "completion_length": 745.0,
+      "epoch": 13.16,
+      "grad_norm": 0.05458225682377815,
+      "kl": 1.0023459196090698,
+      "learning_rate": 4.831990919449806e-06,
+      "loss": 0.0401,
+      "reward": -1.0,
+      "reward_std": 0.0,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 329
+    },
+    {
+      "completion_length": 705.0,
+      "epoch": 13.2,
+      "grad_norm": 0.8195479512214661,
+      "kl": 1.466266393661499,
+      "learning_rate": 4.830019673206997e-06,
+      "loss": 0.0587,
+      "reward": 0.984784722328186,
+      "reward_std": 3.389918565750122,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7222222089767456,
+      "rewards/wrapped_driving_reward": -0.8624374866485596,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 330
+    },
+    {
+      "completion_length": 605.0,
+      "epoch": 13.24,
+      "grad_norm": 0.4812968969345093,
+      "kl": 1.0380324125289917,
+      "learning_rate": 4.828037336897009e-06,
+      "loss": 0.0415,
+      "reward": 2.9450440406799316,
+      "reward_std": 0.32202810049057007,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.07004398852586746,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 331
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 13.28,
+      "grad_norm": 0.5454792380332947,
+      "kl": 1.0666571855545044,
+      "learning_rate": 4.826043919955062e-06,
+      "loss": 0.0427,
+      "reward": 3.0907585620880127,
+      "reward_std": 0.5051584839820862,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.09075860679149628,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 332
+    },
+    {
+      "completion_length": 534.0,
+      "epoch": 13.32,
+      "grad_norm": 0.6665737628936768,
+      "kl": 0.9193040132522583,
+      "learning_rate": 4.824039431869112e-06,
+      "loss": 0.0368,
+      "reward": 3.3314661979675293,
+      "reward_std": 0.48887741565704346,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.875,
+      "rewards/wrapped_driving_reward": 0.5814663171768188,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 333
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 13.36,
+      "grad_norm": 0.8595104217529297,
+      "kl": 0.8587233424186707,
+      "learning_rate": 4.822023882179811e-06,
+      "loss": 0.0343,
+      "reward": 3.173095226287842,
+      "reward_std": 0.4861666262149811,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.6730952262878418,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 334
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 13.4,
+      "grad_norm": 0.661893367767334,
+      "kl": 0.8537664413452148,
+      "learning_rate": 4.8199972804804615e-06,
+      "loss": 0.0342,
+      "reward": 1.0913279056549072,
+      "reward_std": 3.0786876678466797,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9086720943450928,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 335
+    },
+    {
+      "completion_length": 725.0,
+      "epoch": 13.44,
+      "grad_norm": 0.5498742461204529,
+      "kl": 0.8321976065635681,
+      "learning_rate": 4.817959636416969e-06,
+      "loss": 0.0333,
+      "reward": 3.1832504272460938,
+      "reward_std": 0.1209394559264183,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.18325048685073853,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 336
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 13.48,
+      "grad_norm": 1.1294459104537964,
+      "kl": 1.4946460723876953,
+      "learning_rate": 4.815910959687795e-06,
+      "loss": 0.0598,
+      "reward": 2.673915386199951,
+      "reward_std": 0.5557320713996887,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.20108462870121002,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 337
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 13.52,
+      "grad_norm": 0.8809202909469604,
+      "kl": 1.7684038877487183,
+      "learning_rate": 4.8138512600439165e-06,
+      "loss": 0.0707,
+      "reward": 1.2998652458190918,
+      "reward_std": 3.222731351852417,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9501346945762634,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 338
+    },
+    {
+      "completion_length": 528.0,
+      "epoch": 13.56,
+      "grad_norm": 0.6754530072212219,
+      "kl": 0.7506433129310608,
+      "learning_rate": 4.8117805472887706e-06,
+      "loss": 0.03,
+      "reward": 2.930798053741455,
+      "reward_std": 0.7055428624153137,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.4307979345321655,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 339
+    },
+    {
+      "completion_length": 634.0,
+      "epoch": 13.6,
+      "grad_norm": 0.6422954797744751,
+      "kl": 1.1495076417922974,
+      "learning_rate": 4.809698831278217e-06,
+      "loss": 0.046,
+      "reward": 2.855012893676758,
+      "reward_std": 0.6868377923965454,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9375,
+      "rewards/wrapped_driving_reward": 0.16751310229301453,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 340
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 13.64,
+      "grad_norm": 0.8380364775657654,
+      "kl": 0.7830958962440491,
+      "learning_rate": 4.807606121920486e-06,
+      "loss": 0.0313,
+      "reward": 2.7610697746276855,
+      "reward_std": 0.17693300545215607,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9642857313156128,
+      "rewards/wrapped_driving_reward": 0.046783991158008575,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 341
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 13.68,
+      "grad_norm": 0.45640650391578674,
+      "kl": 0.6177964210510254,
+      "learning_rate": 4.80550242917613e-06,
+      "loss": 0.0247,
+      "reward": 0.28038734197616577,
+      "reward_std": 2.0201590061187744,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.949999988079071,
+      "rewards/wrapped_driving_reward": -2.2946126461029053,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 342
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 13.72,
+      "grad_norm": 0.8011844158172607,
+      "kl": 0.8235171437263489,
+      "learning_rate": 4.803387763057981e-06,
+      "loss": 0.0329,
+      "reward": 3.0518081188201904,
+      "reward_std": 0.5246614217758179,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.17680811882019043,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 343
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 13.76,
+      "grad_norm": 0.8099798560142517,
+      "kl": 1.021049976348877,
+      "learning_rate": 4.801262133631101e-06,
+      "loss": 0.0408,
+      "reward": -1.625,
+      "reward_std": 0.25,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 344
+    },
+    {
+      "completion_length": 668.0,
+      "epoch": 13.8,
+      "grad_norm": 0.9226036667823792,
+      "kl": 0.9379998445510864,
+      "learning_rate": 4.799125551012731e-06,
+      "loss": 0.0375,
+      "reward": 2.803008556365967,
+      "reward_std": 0.44537705183029175,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.1780085265636444,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 345
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 13.84,
+      "grad_norm": 0.6966492533683777,
+      "kl": 1.2720391750335693,
+      "learning_rate": 4.796978025372247e-06,
+      "loss": 0.0509,
+      "reward": -1.25,
+      "reward_std": 0.28867512941360474,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.875,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 346
+    },
+    {
+      "completion_length": 547.0,
+      "epoch": 13.88,
+      "grad_norm": 0.5560488700866699,
+      "kl": 0.5576459169387817,
+      "learning_rate": 4.794819566931107e-06,
+      "loss": 0.0223,
+      "reward": 2.6760520935058594,
+      "reward_std": 0.14635096490383148,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.96875,
+      "rewards/wrapped_driving_reward": -0.1676977276802063,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 347
+    },
+    {
+      "completion_length": 606.0,
+      "epoch": 13.92,
+      "grad_norm": 0.1242881491780281,
+      "kl": 0.9755803942680359,
+      "learning_rate": 4.79265018596281e-06,
+      "loss": 0.039,
+      "reward": -1.0,
+      "reward_std": 0.0,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 348
+    },
+    {
+      "completion_length": 476.0,
+      "epoch": 13.96,
+      "grad_norm": 0.9491596221923828,
+      "kl": 0.4472344219684601,
+      "learning_rate": 4.79046989279284e-06,
+      "loss": 0.0179,
+      "reward": 2.4583888053894043,
+      "reward_std": 1.3171792030334473,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.7321428656578064,
+      "rewards/wrapped_driving_reward": 0.2262459248304367,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 349
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 14.0,
+      "grad_norm": 0.43453091382980347,
+      "kl": 0.9528393149375916,
+      "learning_rate": 4.788278697798619e-06,
+      "loss": 0.0381,
+      "reward": 1.0954630374908447,
+      "reward_std": 1.8647366762161255,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.7795369625091553,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 350
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 14.04,
+      "grad_norm": 0.5005083084106445,
+      "kl": 0.8873422145843506,
+      "learning_rate": 4.7860766114094555e-06,
+      "loss": 0.0355,
+      "reward": 2.2282726764678955,
+      "reward_std": 0.6381257176399231,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.27172738313674927,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 351
+    },
+    {
+      "completion_length": 736.0,
+      "epoch": 14.08,
+      "grad_norm": 0.6733227968215942,
+      "kl": 0.902958333492279,
+      "learning_rate": 4.783863644106502e-06,
+      "loss": 0.0361,
+      "reward": 2.6390929222106934,
+      "reward_std": 0.7452251315116882,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8999999761581421,
+      "rewards/wrapped_driving_reward": 0.11409273743629456,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 352
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 14.12,
+      "grad_norm": 0.3890395164489746,
+      "kl": 0.21702060103416443,
+      "learning_rate": 4.781639806422699e-06,
+      "loss": 0.0087,
+      "reward": 2.652873992919922,
+      "reward_std": 0.42699992656707764,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.027873992919921875,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 353
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 14.16,
+      "grad_norm": 2.0187084674835205,
+      "kl": 1.372572660446167,
+      "learning_rate": 4.779405108942722e-06,
+      "loss": 0.0549,
+      "reward": 2.4973678588867188,
+      "reward_std": 0.37870702147483826,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.002632094081491232,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 354
+    },
+    {
+      "completion_length": 528.0,
+      "epoch": 14.2,
+      "grad_norm": 0.6127402782440186,
+      "kl": 0.7212586998939514,
+      "learning_rate": 4.77715956230294e-06,
+      "loss": 0.0288,
+      "reward": -1.0456349849700928,
+      "reward_std": 0.055128760635852814,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.954365074634552,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 355
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 14.24,
+      "grad_norm": 0.7625749111175537,
+      "kl": 1.294076681137085,
+      "learning_rate": 4.774903177191358e-06,
+      "loss": 0.0518,
+      "reward": 2.880377769470215,
+      "reward_std": 0.6150814890861511,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.13037782907485962,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 356
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 14.28,
+      "grad_norm": 1.118186116218567,
+      "kl": 1.3434715270996094,
+      "learning_rate": 4.77263596434757e-06,
+      "loss": 0.0537,
+      "reward": 3.2224440574645996,
+      "reward_std": 0.4224244952201843,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9642857313156128,
+      "rewards/wrapped_driving_reward": 0.38315847516059875,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 357
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 14.32,
+      "grad_norm": 0.5385660529136658,
+      "kl": 1.1639105081558228,
+      "learning_rate": 4.770357934562704e-06,
+      "loss": 0.0466,
+      "reward": 3.100466728210449,
+      "reward_std": 0.24647371470928192,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9375,
+      "rewards/wrapped_driving_reward": 0.16296681761741638,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 358
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 14.36,
+      "grad_norm": 0.7925487756729126,
+      "kl": 1.1513047218322754,
+      "learning_rate": 4.7680690986793734e-06,
+      "loss": 0.0461,
+      "reward": -2.125,
+      "reward_std": 1.0307763814926147,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.625,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 359
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 14.4,
+      "grad_norm": 0.5221444368362427,
+      "kl": 0.9014317989349365,
+      "learning_rate": 4.765769467591626e-06,
+      "loss": 0.0361,
+      "reward": 3.1284172534942627,
+      "reward_std": 0.13471664488315582,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.1284172236919403,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 360
+    },
+    {
+      "completion_length": 619.0,
+      "epoch": 14.44,
+      "grad_norm": 0.7254578471183777,
+      "kl": 1.2201420068740845,
+      "learning_rate": 4.7634590522448886e-06,
+      "loss": 0.0488,
+      "reward": -1.28125,
+      "reward_std": 0.4827762544155121,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.96875,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 361
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 14.48,
+      "grad_norm": 0.6827868819236755,
+      "kl": 1.4321162700653076,
+      "learning_rate": 4.761137863635921e-06,
+      "loss": 0.0573,
+      "reward": 3.0515081882476807,
+      "reward_std": 0.42571601271629333,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.051508113741874695,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 362
+    },
+    {
+      "completion_length": 589.0,
+      "epoch": 14.52,
+      "grad_norm": 0.8641681671142578,
+      "kl": 1.0050054788589478,
+      "learning_rate": 4.758805912812755e-06,
+      "loss": 0.0402,
+      "reward": 3.0080392360687256,
+      "reward_std": 0.8377665281295776,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.25803929567337036,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 363
+    },
+    {
+      "completion_length": 666.0,
+      "epoch": 14.56,
+      "grad_norm": 0.9824696779251099,
+      "kl": 0.7488256692886353,
+      "learning_rate": 4.7564632108746524e-06,
+      "loss": 0.03,
+      "reward": -1.25,
+      "reward_std": 0.28867512941360474,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 364
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 14.6,
+      "grad_norm": 0.3840194344520569,
+      "kl": 1.0556010007858276,
+      "learning_rate": 4.75410976897204e-06,
+      "loss": 0.0422,
+      "reward": 3.167947292327881,
+      "reward_std": 0.44172561168670654,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.1679472178220749,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 365
+    },
+    {
+      "completion_length": 630.0,
+      "epoch": 14.64,
+      "grad_norm": 0.4588969647884369,
+      "kl": 0.5971285104751587,
+      "learning_rate": 4.7517455983064694e-06,
+      "loss": 0.0239,
+      "reward": -1.0227272510528564,
+      "reward_std": 0.04545450210571289,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9772727489471436,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 366
+    },
+    {
+      "completion_length": 730.0,
+      "epoch": 14.68,
+      "grad_norm": 0.834652841091156,
+      "kl": 1.3273468017578125,
+      "learning_rate": 4.7493707101305545e-06,
+      "loss": 0.0531,
+      "reward": 2.759453773498535,
+      "reward_std": 0.5071139335632324,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9583333134651184,
+      "rewards/wrapped_driving_reward": 0.1761203557252884,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 367
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 14.72,
+      "grad_norm": 0.5011023879051208,
+      "kl": 1.0096584558486938,
+      "learning_rate": 4.746985115747918e-06,
+      "loss": 0.0404,
+      "reward": 2.915914535522461,
+      "reward_std": 0.5742588043212891,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9642857313156128,
+      "rewards/wrapped_driving_reward": 0.7016288638114929,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 368
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 14.76,
+      "grad_norm": 0.6958830952644348,
+      "kl": 1.282517910003662,
+      "learning_rate": 4.744588826513145e-06,
+      "loss": 0.0513,
+      "reward": 2.461658477783203,
+      "reward_std": 1.062435269355774,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.2883416414260864,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 369
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 14.8,
+      "grad_norm": 0.5009940266609192,
+      "kl": 0.7665615081787109,
+      "learning_rate": 4.742181853831721e-06,
+      "loss": 0.0307,
+      "reward": 3.0534534454345703,
+      "reward_std": 0.42255762219429016,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.949999988079071,
+      "rewards/wrapped_driving_reward": 0.10345339775085449,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 370
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 14.84,
+      "grad_norm": 0.3792387545108795,
+      "kl": 0.9433217644691467,
+      "learning_rate": 4.739764209159984e-06,
+      "loss": 0.0377,
+      "reward": 1.4562020301818848,
+      "reward_std": 3.3310694694519043,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.7937980890274048,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 371
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 14.88,
+      "grad_norm": 0.6532329320907593,
+      "kl": 1.3375815153121948,
+      "learning_rate": 4.737335904005063e-06,
+      "loss": 0.0535,
+      "reward": 1.3177659511566162,
+      "reward_std": 2.252401113510132,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.1822340488433838,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 372
+    },
+    {
+      "completion_length": 564.0,
+      "epoch": 14.92,
+      "grad_norm": 0.6528195738792419,
+      "kl": 0.9802216291427612,
+      "learning_rate": 4.734896949924831e-06,
+      "loss": 0.0392,
+      "reward": 1.9856492280960083,
+      "reward_std": 0.9138504266738892,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.875,
+      "rewards/wrapped_driving_reward": -0.2643508017063141,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 373
+    },
+    {
+      "completion_length": 560.0,
+      "epoch": 14.96,
+      "grad_norm": 0.613873302936554,
+      "kl": 1.106533169746399,
+      "learning_rate": 4.732447358527843e-06,
+      "loss": 0.0443,
+      "reward": 3.070741891860962,
+      "reward_std": 0.6192570328712463,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.19574186205863953,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 374
+    },
+    {
+      "completion_length": 742.0,
+      "epoch": 15.0,
+      "grad_norm": 0.5938997268676758,
+      "kl": 1.087781310081482,
+      "learning_rate": 4.729987141473286e-06,
+      "loss": 0.0435,
+      "reward": 3.193850040435791,
+      "reward_std": 0.5507637858390808,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.949999988079071,
+      "rewards/wrapped_driving_reward": 0.6188501119613647,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 375
+    },
+    {
+      "completion_length": 527.0,
+      "epoch": 15.04,
+      "grad_norm": 0.44303637742996216,
+      "kl": 0.551745593547821,
+      "learning_rate": 4.72751631047092e-06,
+      "loss": 0.0221,
+      "reward": 2.2447049617767334,
+      "reward_std": 0.40859082341194153,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.3802950978279114,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 376
+    },
+    {
+      "completion_length": 504.0,
+      "epoch": 15.08,
+      "grad_norm": 0.4766407310962677,
+      "kl": 0.5846800804138184,
+      "learning_rate": 4.725034877281025e-06,
+      "loss": 0.0234,
+      "reward": 2.4882283210754395,
+      "reward_std": 0.9573307633399963,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.5117717385292053,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 377
+    },
+    {
+      "completion_length": 741.0,
+      "epoch": 15.12,
+      "grad_norm": 1.0294671058654785,
+      "kl": 0.909957230091095,
+      "learning_rate": 4.7225428537143414e-06,
+      "loss": 0.0364,
+      "reward": 2.707183361053467,
+      "reward_std": 0.2954871654510498,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9615384340286255,
+      "rewards/wrapped_driving_reward": 0.12064509093761444,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 378
+    },
+    {
+      "completion_length": 725.0,
+      "epoch": 15.16,
+      "grad_norm": 1.9352197647094727,
+      "kl": 1.4188382625579834,
+      "learning_rate": 4.720040251632019e-06,
+      "loss": 0.0568,
+      "reward": 2.6802406311035156,
+      "reward_std": 0.7824314832687378,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8434343338012695,
+      "rewards/wrapped_driving_reward": 0.08680635690689087,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 379
+    },
+    {
+      "completion_length": 680.0,
+      "epoch": 15.2,
+      "grad_norm": 1.030442237854004,
+      "kl": 0.9825220704078674,
+      "learning_rate": 4.717527082945555e-06,
+      "loss": 0.0393,
+      "reward": 3.433924436569214,
+      "reward_std": 0.5535728335380554,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.6839244961738586,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 380
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 15.24,
+      "grad_norm": 0.6829676032066345,
+      "kl": 1.046954870223999,
+      "learning_rate": 4.715003359616741e-06,
+      "loss": 0.0419,
+      "reward": -1.125,
+      "reward_std": 0.25,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 381
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 15.28,
+      "grad_norm": 0.5402231812477112,
+      "kl": 0.7217246294021606,
+      "learning_rate": 4.712469093657605e-06,
+      "loss": 0.0289,
+      "reward": 2.7015037536621094,
+      "reward_std": 0.6882119178771973,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.17349615693092346,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 382
+    },
+    {
+      "completion_length": 656.0,
+      "epoch": 15.32,
+      "grad_norm": 0.8153801560401917,
+      "kl": 1.4001483917236328,
+      "learning_rate": 4.709924297130354e-06,
+      "loss": 0.056,
+      "reward": -1.2777777910232544,
+      "reward_std": 0.48432207107543945,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9722222089767456,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 383
+    },
+    {
+      "completion_length": 663.0,
+      "epoch": 15.36,
+      "grad_norm": 0.3903270661830902,
+      "kl": 1.1370939016342163,
+      "learning_rate": 4.707368982147318e-06,
+      "loss": 0.0455,
+      "reward": 0.018291592597961426,
+      "reward_std": 2.9197871685028076,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7250000238418579,
+      "rewards/wrapped_driving_reward": -2.2067084312438965,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 384
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 15.4,
+      "grad_norm": 0.49858081340789795,
+      "kl": 0.7010886073112488,
+      "learning_rate": 4.704803160870888e-06,
+      "loss": 0.028,
+      "reward": 2.679309844970703,
+      "reward_std": 0.4782131314277649,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.07069025933742523,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 385
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 15.44,
+      "grad_norm": 0.6627248525619507,
+      "kl": 1.3270666599273682,
+      "learning_rate": 4.702226845513465e-06,
+      "loss": 0.0531,
+      "reward": -1.25,
+      "reward_std": 0.5,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 386
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 15.48,
+      "grad_norm": 0.7167011499404907,
+      "kl": 1.1115460395812988,
+      "learning_rate": 4.699640048337394e-06,
+      "loss": 0.0445,
+      "reward": -1.375,
+      "reward_std": 0.4787135720252991,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 387
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 15.52,
+      "grad_norm": 0.33501261472702026,
+      "kl": 1.264574408531189,
+      "learning_rate": 4.697042781654913e-06,
+      "loss": 0.0506,
+      "reward": 2.486931324005127,
+      "reward_std": 0.23576277494430542,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.1380685567855835,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 388
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 15.56,
+      "grad_norm": 0.5400851368904114,
+      "kl": 1.1933934688568115,
+      "learning_rate": 4.694435057828092e-06,
+      "loss": 0.0477,
+      "reward": 1.214620590209961,
+      "reward_std": 2.837777614593506,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.0353795289993286,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 389
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 15.6,
+      "grad_norm": 0.44655412435531616,
+      "kl": 0.6380826830863953,
+      "learning_rate": 4.69181688926877e-06,
+      "loss": 0.0255,
+      "reward": 3.148019313812256,
+      "reward_std": 0.17191006243228912,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.1480191946029663,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 390
+    },
+    {
+      "completion_length": 690.0,
+      "epoch": 15.64,
+      "grad_norm": 0.6045768857002258,
+      "kl": 1.1987158060073853,
+      "learning_rate": 4.6891882884384994e-06,
+      "loss": 0.0479,
+      "reward": 1.2684307098388672,
+      "reward_std": 2.863405704498291,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.71875,
+      "rewards/wrapped_driving_reward": -0.825319230556488,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 391
+    },
+    {
+      "completion_length": 741.0,
+      "epoch": 15.68,
+      "grad_norm": 0.4550888240337372,
+      "kl": 1.3106390237808228,
+      "learning_rate": 4.68654926784849e-06,
+      "loss": 0.0524,
+      "reward": 2.7248356342315674,
+      "reward_std": 0.2842490077018738,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.22483548521995544,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 392
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 15.72,
+      "grad_norm": 0.49233898520469666,
+      "kl": 1.008697509765625,
+      "learning_rate": 4.683899840059543e-06,
+      "loss": 0.0403,
+      "reward": -1.6673097610473633,
+      "reward_std": 1.2686203718185425,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -3.6673097610473633,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 393
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 15.76,
+      "grad_norm": 0.5187327861785889,
+      "kl": 1.2990044355392456,
+      "learning_rate": 4.681240017681994e-06,
+      "loss": 0.052,
+      "reward": -1.125,
+      "reward_std": 0.25,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 394
+    },
+    {
+      "completion_length": 506.0,
+      "epoch": 15.8,
+      "grad_norm": 0.6884944438934326,
+      "kl": 0.7200514078140259,
+      "learning_rate": 4.678569813375654e-06,
+      "loss": 0.0288,
+      "reward": 3.613212823867798,
+      "reward_std": 0.14553166925907135,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.6132129430770874,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 395
+    },
+    {
+      "completion_length": 635.0,
+      "epoch": 15.84,
+      "grad_norm": 0.5450335741043091,
+      "kl": 1.369678258895874,
+      "learning_rate": 4.675889239849749e-06,
+      "loss": 0.0548,
+      "reward": 2.0136775970458984,
+      "reward_std": 1.1577099561691284,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.6113223433494568,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 396
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 15.88,
+      "grad_norm": 0.4165554642677307,
+      "kl": 1.6805375814437866,
+      "learning_rate": 4.67319830986286e-06,
+      "loss": 0.0672,
+      "reward": 3.094829559326172,
+      "reward_std": 0.6602194309234619,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.3448294997215271,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 397
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 15.92,
+      "grad_norm": 0.4113013744354248,
+      "kl": 0.983104944229126,
+      "learning_rate": 4.670497036222856e-06,
+      "loss": 0.0393,
+      "reward": 2.9975690841674805,
+      "reward_std": 0.22446000576019287,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.00243115215562284,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 398
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 15.96,
+      "grad_norm": 0.8818913102149963,
+      "kl": 1.1384490728378296,
+      "learning_rate": 4.667785431786843e-06,
+      "loss": 0.0455,
+      "reward": 3.1137232780456543,
+      "reward_std": 0.4699656367301941,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.949999988079071,
+      "rewards/wrapped_driving_reward": 0.538723349571228,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 399
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 16.0,
+      "grad_norm": 0.41796690225601196,
+      "kl": 1.118818759918213,
+      "learning_rate": 4.665063509461098e-06,
+      "loss": 0.0448,
+      "reward": 2.9344351291656494,
+      "reward_std": 0.2725065052509308,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.05943508818745613,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 400
+    },
+    {
+      "completion_length": 510.0,
+      "epoch": 16.04,
+      "grad_norm": 0.5188367366790771,
+      "kl": 0.7825106382369995,
+      "learning_rate": 4.662331282201002e-06,
+      "loss": 0.0313,
+      "reward": 2.8269712924957275,
+      "reward_std": 0.5569232106208801,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9166666865348816,
+      "rewards/wrapped_driving_reward": 0.03530450910329819,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 401
+    },
+    {
+      "completion_length": 673.0,
+      "epoch": 16.08,
+      "grad_norm": 0.06149033084511757,
+      "kl": 0.8966481685638428,
+      "learning_rate": 4.65958876301099e-06,
+      "loss": 0.0359,
+      "reward": -1.0,
+      "reward_std": 0.0,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 402
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 16.12,
+      "grad_norm": 0.5415775775909424,
+      "kl": 1.1003456115722656,
+      "learning_rate": 4.65683596494448e-06,
+      "loss": 0.044,
+      "reward": -1.3068182468414307,
+      "reward_std": 0.24159422516822815,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9431818127632141,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 403
+    },
+    {
+      "completion_length": 620.0,
+      "epoch": 16.16,
+      "grad_norm": 0.42184168100357056,
+      "kl": 1.1518326997756958,
+      "learning_rate": 4.654072901103815e-06,
+      "loss": 0.0461,
+      "reward": 3.293445348739624,
+      "reward_std": 0.3596523404121399,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.29344528913497925,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 404
+    },
+    {
+      "completion_length": 738.0,
+      "epoch": 16.2,
+      "grad_norm": 0.5668161511421204,
+      "kl": 1.5937199592590332,
+      "learning_rate": 4.651299584640198e-06,
+      "loss": 0.0637,
+      "reward": -1.2777777910232544,
+      "reward_std": 0.5555555820465088,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9722222089767456,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 405
+    },
+    {
+      "completion_length": 582.0,
+      "epoch": 16.24,
+      "grad_norm": 0.46643057465553284,
+      "kl": 1.0229408740997314,
+      "learning_rate": 4.648516028753632e-06,
+      "loss": 0.0409,
+      "reward": 2.7324419021606445,
+      "reward_std": 0.4210711717605591,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.96875,
+      "rewards/wrapped_driving_reward": -0.23630811274051666,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 406
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 16.28,
+      "grad_norm": 0.7909368872642517,
+      "kl": 1.144740104675293,
+      "learning_rate": 4.645722246692856e-06,
+      "loss": 0.0458,
+      "reward": -1.125,
+      "reward_std": 0.25,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.875,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 407
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 16.32,
+      "grad_norm": 0.40184658765792847,
+      "kl": 1.196555733680725,
+      "learning_rate": 4.642918251755281e-06,
+      "loss": 0.0479,
+      "reward": 2.915738582611084,
+      "reward_std": 0.2729151248931885,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.08426124602556229,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 408
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 16.36,
+      "grad_norm": 0.38038399815559387,
+      "kl": 0.5281871557235718,
+      "learning_rate": 4.6401040572869295e-06,
+      "loss": 0.0211,
+      "reward": 2.0221829414367676,
+      "reward_std": 0.7135373950004578,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.7278171181678772,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 409
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 16.4,
+      "grad_norm": 0.6318146586418152,
+      "kl": 1.4290169477462769,
+      "learning_rate": 4.637279676682367e-06,
+      "loss": 0.0572,
+      "reward": 2.355543851852417,
+      "reward_std": 1.3740731477737427,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.3944561779499054,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 410
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 16.44,
+      "grad_norm": 0.8606741428375244,
+      "kl": 1.26045560836792,
+      "learning_rate": 4.634445123384644e-06,
+      "loss": 0.0504,
+      "reward": 2.573911190032959,
+      "reward_std": 0.6645153760910034,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.07391127943992615,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 411
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 16.48,
+      "grad_norm": 0.40005964040756226,
+      "kl": 1.007511019706726,
+      "learning_rate": 4.631600410885231e-06,
+      "loss": 0.0403,
+      "reward": 0.41647136211395264,
+      "reward_std": 1.9502681493759155,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.949999988079071,
+      "rewards/wrapped_driving_reward": -2.0335285663604736,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 412
+    },
+    {
+      "completion_length": 593.0,
+      "epoch": 16.52,
+      "grad_norm": 0.5277693271636963,
+      "kl": 1.2088240385055542,
+      "learning_rate": 4.6287455527239475e-06,
+      "loss": 0.0484,
+      "reward": 2.7194738388061523,
+      "reward_std": 0.4919568598270416,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.09447365999221802,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 413
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 16.56,
+      "grad_norm": 0.4249366223812103,
+      "kl": 0.8536944389343262,
+      "learning_rate": 4.625880562488908e-06,
+      "loss": 0.0341,
+      "reward": 3.555159091949463,
+      "reward_std": 0.2293834686279297,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.6801592111587524,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 414
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 16.6,
+      "grad_norm": 0.6364783644676208,
+      "kl": 1.720573902130127,
+      "learning_rate": 4.623005453816447e-06,
+      "loss": 0.0688,
+      "reward": 2.6921558380126953,
+      "reward_std": 0.33053314685821533,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.06715589016675949,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 415
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 16.64,
+      "grad_norm": 0.5456056594848633,
+      "kl": 1.410339117050171,
+      "learning_rate": 4.620120240391065e-06,
+      "loss": 0.0564,
+      "reward": 2.0680339336395264,
+      "reward_std": 1.2569472789764404,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.681966245174408,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 416
+    },
+    {
+      "completion_length": 590.0,
+      "epoch": 16.68,
+      "grad_norm": 0.5361428260803223,
+      "kl": 0.9750890731811523,
+      "learning_rate": 4.617224935945354e-06,
+      "loss": 0.039,
+      "reward": 3.0615792274475098,
+      "reward_std": 0.41482946276664734,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.18657910823822021,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 417
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 16.72,
+      "grad_norm": 0.3962303102016449,
+      "kl": 1.1152567863464355,
+      "learning_rate": 4.614319554259934e-06,
+      "loss": 0.0446,
+      "reward": 2.119654417037964,
+      "reward_std": 0.8136077523231506,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.38034552335739136,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 418
+    },
+    {
+      "completion_length": 650.0,
+      "epoch": 16.76,
+      "grad_norm": 0.4923894703388214,
+      "kl": 0.6831648945808411,
+      "learning_rate": 4.611404109163392e-06,
+      "loss": 0.0273,
+      "reward": 2.7168376445770264,
+      "reward_std": 0.5373032093048096,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.03316231817007065,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 419
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 16.8,
+      "grad_norm": 0.5763388872146606,
+      "kl": 1.2227815389633179,
+      "learning_rate": 4.608478614532215e-06,
+      "loss": 0.0489,
+      "reward": 3.023646354675293,
+      "reward_std": 0.09265323728322983,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.14864633977413177,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 420
+    },
+    {
+      "completion_length": 472.0,
+      "epoch": 16.84,
+      "grad_norm": 0.7298195958137512,
+      "kl": 1.2331024408340454,
+      "learning_rate": 4.605543084290716e-06,
+      "loss": 0.0493,
+      "reward": 3.369495153427124,
+      "reward_std": 0.40679696202278137,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.49449509382247925,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 421
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 16.88,
+      "grad_norm": 0.5339130163192749,
+      "kl": 1.6134005784988403,
+      "learning_rate": 4.602597532410982e-06,
+      "loss": 0.0645,
+      "reward": 3.0414419174194336,
+      "reward_std": 0.6409664154052734,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9750000238418579,
+      "rewards/wrapped_driving_reward": 0.31644195318222046,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 422
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 16.92,
+      "grad_norm": 0.43448933959007263,
+      "kl": 1.194190263748169,
+      "learning_rate": 4.599641972912791e-06,
+      "loss": 0.0478,
+      "reward": 1.0178461074829102,
+      "reward_std": 3.0411243438720703,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.699999988079071,
+      "rewards/wrapped_driving_reward": -0.9321538805961609,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 423
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 16.96,
+      "grad_norm": 0.65577232837677,
+      "kl": 1.1475871801376343,
+      "learning_rate": 4.596676419863561e-06,
+      "loss": 0.0459,
+      "reward": 3.7684073448181152,
+      "reward_std": 0.06588174402713776,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.7684074640274048,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 424
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 17.0,
+      "grad_norm": 0.4012209177017212,
+      "kl": 1.4444128274917603,
+      "learning_rate": 4.59370088737827e-06,
+      "loss": 0.0578,
+      "reward": -1.0416667461395264,
+      "reward_std": 0.08333337306976318,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9583333134651184,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 425
+    },
+    {
+      "completion_length": 520.0,
+      "epoch": 17.04,
+      "grad_norm": 0.5853270888328552,
+      "kl": 1.0507853031158447,
+      "learning_rate": 4.590715389619399e-06,
+      "loss": 0.042,
+      "reward": 2.998222827911377,
+      "reward_std": 0.3459242880344391,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.942307710647583,
+      "rewards/wrapped_driving_reward": 0.18091517686843872,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 426
+    },
+    {
+      "completion_length": 517.0,
+      "epoch": 17.08,
+      "grad_norm": 0.47325876355171204,
+      "kl": 0.7153458595275879,
+      "learning_rate": 4.587719940796858e-06,
+      "loss": 0.0286,
+      "reward": 3.0170512199401855,
+      "reward_std": 0.35850608348846436,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.01705138385295868,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 427
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 17.12,
+      "grad_norm": 0.46970948576927185,
+      "kl": 0.750531792640686,
+      "learning_rate": 4.584714555167921e-06,
+      "loss": 0.03,
+      "reward": 1.5067301988601685,
+      "reward_std": 2.1269452571868896,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.493269681930542,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 428
+    },
+    {
+      "completion_length": 690.0,
+      "epoch": 17.16,
+      "grad_norm": 0.4938046336174011,
+      "kl": 0.9764611124992371,
+      "learning_rate": 4.581699247037157e-06,
+      "loss": 0.0391,
+      "reward": 3.021587371826172,
+      "reward_std": 0.5245987772941589,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.984375,
+      "rewards/wrapped_driving_reward": 0.28721246123313904,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 429
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 17.2,
+      "grad_norm": 0.40477994084358215,
+      "kl": 1.7532377243041992,
+      "learning_rate": 4.578674030756364e-06,
+      "loss": 0.0701,
+      "reward": 2.8085033893585205,
+      "reward_std": 0.7948499917984009,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.875,
+      "rewards/wrapped_driving_reward": 0.30850329995155334,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 430
+    },
+    {
+      "completion_length": 444.0,
+      "epoch": 17.24,
+      "grad_norm": 0.7702034115791321,
+      "kl": 0.8725172877311707,
+      "learning_rate": 4.5756389207244965e-06,
+      "loss": 0.0349,
+      "reward": 2.688070774078369,
+      "reward_std": 0.7989228963851929,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.90625,
+      "rewards/wrapped_driving_reward": 0.15682078897953033,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 431
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 17.28,
+      "grad_norm": 0.5551993250846863,
+      "kl": 0.5444793105125427,
+      "learning_rate": 4.572593931387604e-06,
+      "loss": 0.0218,
+      "reward": -1.625,
+      "reward_std": 0.4787135720252991,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 432
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 17.32,
+      "grad_norm": 0.4201129078865051,
+      "kl": 1.078975796699524,
+      "learning_rate": 4.569539077238756e-06,
+      "loss": 0.0432,
+      "reward": 1.402123212814331,
+      "reward_std": 2.957822561264038,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.737500011920929,
+      "rewards/wrapped_driving_reward": -0.8353768587112427,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 433
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 17.36,
+      "grad_norm": 0.47643131017684937,
+      "kl": 1.2594619989395142,
+      "learning_rate": 4.566474372817971e-06,
+      "loss": 0.0504,
+      "reward": 3.108989715576172,
+      "reward_std": 0.7875818014144897,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.23398981988430023,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 434
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 17.4,
+      "grad_norm": 0.4843199551105499,
+      "kl": 0.4984547793865204,
+      "learning_rate": 4.5633998327121595e-06,
+      "loss": 0.0199,
+      "reward": 0.8982076644897461,
+      "reward_std": 3.3170034885406494,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7321428656578064,
+      "rewards/wrapped_driving_reward": -1.083935260772705,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 435
+    },
+    {
+      "completion_length": 544.0,
+      "epoch": 17.44,
+      "grad_norm": 0.49070167541503906,
+      "kl": 0.45354408025741577,
+      "learning_rate": 4.560315471555039e-06,
+      "loss": 0.0181,
+      "reward": 1.0802078247070312,
+      "reward_std": 2.0147366523742676,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.9197921752929688,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 436
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 17.48,
+      "grad_norm": 0.4404445290565491,
+      "kl": 1.0045150518417358,
+      "learning_rate": 4.557221304027077e-06,
+      "loss": 0.0402,
+      "reward": 2.6005172729492188,
+      "reward_std": 0.8598195910453796,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": 0.10051736980676651,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 437
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 17.52,
+      "grad_norm": 0.501968264579773,
+      "kl": 1.408727765083313,
+      "learning_rate": 4.55411734485541e-06,
+      "loss": 0.0563,
+      "reward": 2.800401210784912,
+      "reward_std": 0.44490158557891846,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": 0.05040114372968674,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 438
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 17.56,
+      "grad_norm": 0.5490036010742188,
+      "kl": 1.1226475238800049,
+      "learning_rate": 4.551003608813784e-06,
+      "loss": 0.0449,
+      "reward": 0.49088820815086365,
+      "reward_std": 1.7767810821533203,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -2.3841118812561035,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 439
+    },
+    {
+      "completion_length": 716.0,
+      "epoch": 17.6,
+      "grad_norm": 0.636411726474762,
+      "kl": 1.195053219795227,
+      "learning_rate": 4.54788011072248e-06,
+      "loss": 0.0478,
+      "reward": 1.8398276567459106,
+      "reward_std": 1.4837794303894043,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.1601723432540894,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 440
+    },
+    {
+      "completion_length": 511.0,
+      "epoch": 17.64,
+      "grad_norm": 0.6527817845344543,
+      "kl": 0.8692322969436646,
+      "learning_rate": 4.544746865448239e-06,
+      "loss": 0.0348,
+      "reward": -1.2857142686843872,
+      "reward_std": 0.48092880845069885,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9642857313156128,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 441
+    },
+    {
+      "completion_length": 594.0,
+      "epoch": 17.68,
+      "grad_norm": 0.6898093223571777,
+      "kl": 1.2306536436080933,
+      "learning_rate": 4.541603887904198e-06,
+      "loss": 0.0492,
+      "reward": -1.0625,
+      "reward_std": 0.125,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9375,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 442
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 17.72,
+      "grad_norm": 0.3844476640224457,
+      "kl": 0.9789438247680664,
+      "learning_rate": 4.538451193049814e-06,
+      "loss": 0.0392,
+      "reward": 2.9422659873962402,
+      "reward_std": 0.6176936030387878,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9722222089767456,
+      "rewards/wrapped_driving_reward": -0.02995637059211731,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 443
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 17.76,
+      "grad_norm": 0.3810977339744568,
+      "kl": 0.8543868660926819,
+      "learning_rate": 4.535288795890799e-06,
+      "loss": 0.0342,
+      "reward": -1.75,
+      "reward_std": 1.5,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 444
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 17.8,
+      "grad_norm": 0.5191684365272522,
+      "kl": 0.4153221845626831,
+      "learning_rate": 4.532116711479039e-06,
+      "loss": 0.0166,
+      "reward": 2.4012069702148438,
+      "reward_std": 0.41621556878089905,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.2237929105758667,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 445
+    },
+    {
+      "completion_length": 630.0,
+      "epoch": 17.84,
+      "grad_norm": 0.5015482902526855,
+      "kl": 0.5795023441314697,
+      "learning_rate": 4.528934954912531e-06,
+      "loss": 0.0232,
+      "reward": 2.996004819869995,
+      "reward_std": 0.16818639636039734,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.984375,
+      "rewards/wrapped_driving_reward": 0.011629827320575714,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 446
+    },
+    {
+      "completion_length": 556.0,
+      "epoch": 17.88,
+      "grad_norm": 0.9043763279914856,
+      "kl": 1.600888967514038,
+      "learning_rate": 4.525743541335309e-06,
+      "loss": 0.064,
+      "reward": 3.2926249504089355,
+      "reward_std": 0.6622192859649658,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.417624831199646,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 447
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 17.92,
+      "grad_norm": 0.5094466209411621,
+      "kl": 0.9114040732383728,
+      "learning_rate": 4.522542485937369e-06,
+      "loss": 0.0365,
+      "reward": -1.2035714387893677,
+      "reward_std": 0.21614274382591248,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9214285612106323,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 448
+    },
+    {
+      "completion_length": 653.0,
+      "epoch": 17.96,
+      "grad_norm": 0.6721723675727844,
+      "kl": 1.3184740543365479,
+      "learning_rate": 4.519331803954599e-06,
+      "loss": 0.0527,
+      "reward": 2.8614964485168457,
+      "reward_std": 0.2383938878774643,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.1114964485168457,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 449
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 18.0,
+      "grad_norm": 0.4677048623561859,
+      "kl": 0.28004857897758484,
+      "learning_rate": 4.516111510668707e-06,
+      "loss": 0.0112,
+      "reward": -0.9574483036994934,
+      "reward_std": 3.5327820777893066,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -2.2074482440948486,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 450
+    },
+    {
+      "completion_length": 723.0,
+      "epoch": 18.04,
+      "grad_norm": 0.8021937608718872,
+      "kl": 1.2376339435577393,
+      "learning_rate": 4.512881621407146e-06,
+      "loss": 0.0495,
+      "reward": 2.9628384113311768,
+      "reward_std": 0.32038041949272156,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9226190447807312,
+      "rewards/wrapped_driving_reward": 0.16521935164928436,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 451
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 18.08,
+      "grad_norm": 0.6682026386260986,
+      "kl": 0.9013283252716064,
+      "learning_rate": 4.509642151543043e-06,
+      "loss": 0.0361,
+      "reward": 2.2376132011413574,
+      "reward_std": 1.6770848035812378,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.5123868584632874,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 452
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 18.12,
+      "grad_norm": 0.3643920123577118,
+      "kl": 0.7929832935333252,
+      "learning_rate": 4.506393116495128e-06,
+      "loss": 0.0317,
+      "reward": 2.7846744060516357,
+      "reward_std": 0.17307978868484497,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9027777910232544,
+      "rewards/wrapped_driving_reward": 0.006896574050188065,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 453
+    },
+    {
+      "completion_length": 734.0,
+      "epoch": 18.16,
+      "grad_norm": 1.3546857833862305,
+      "kl": 1.7289873361587524,
+      "learning_rate": 4.503134531727652e-06,
+      "loss": 0.0692,
+      "reward": 3.0436387062072754,
+      "reward_std": 0.573936939239502,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9083333015441895,
+      "rewards/wrapped_driving_reward": 0.3853055536746979,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 454
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 18.2,
+      "grad_norm": 0.9518548250198364,
+      "kl": 1.1346514225006104,
+      "learning_rate": 4.499866412750324e-06,
+      "loss": 0.0454,
+      "reward": 2.846243381500244,
+      "reward_std": 0.40401574969291687,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.90625,
+      "rewards/wrapped_driving_reward": 0.18999344110488892,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 455
+    },
+    {
+      "completion_length": 528.0,
+      "epoch": 18.24,
+      "grad_norm": 0.5974892973899841,
+      "kl": 1.403336763381958,
+      "learning_rate": 4.496588775118232e-06,
+      "loss": 0.0561,
+      "reward": 2.7360715866088867,
+      "reward_std": 0.7795047760009766,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.2360716164112091,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 456
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 18.28,
+      "grad_norm": 0.7698376774787903,
+      "kl": 0.9995352625846863,
+      "learning_rate": 4.493301634431768e-06,
+      "loss": 0.04,
+      "reward": 0.8301829099655151,
+      "reward_std": 3.2255442142486572,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.692307710647583,
+      "rewards/wrapped_driving_reward": -1.2371246814727783,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 457
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 18.32,
+      "grad_norm": 0.7532805800437927,
+      "kl": 1.3239738941192627,
+      "learning_rate": 4.490005006336555e-06,
+      "loss": 0.053,
+      "reward": 1.7292661666870117,
+      "reward_std": 1.0531554222106934,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.2707338333129883,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 458
+    },
+    {
+      "completion_length": 609.0,
+      "epoch": 18.36,
+      "grad_norm": 1.3088675737380981,
+      "kl": 1.1671568155288696,
+      "learning_rate": 4.486698906523375e-06,
+      "loss": 0.0467,
+      "reward": -1.375,
+      "reward_std": 0.25,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 459
+    },
+    {
+      "completion_length": 565.0,
+      "epoch": 18.4,
+      "grad_norm": 0.5267041325569153,
+      "kl": 0.8863846659660339,
+      "learning_rate": 4.4833833507280884e-06,
+      "loss": 0.0355,
+      "reward": 1.512794852256775,
+      "reward_std": 3.00907826423645,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9872051477432251,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 460
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 18.44,
+      "grad_norm": 0.4046444296836853,
+      "kl": 1.289667010307312,
+      "learning_rate": 4.4800583547315654e-06,
+      "loss": 0.0516,
+      "reward": -1.0499999523162842,
+      "reward_std": 0.10000002384185791,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.949999988079071,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 461
+    },
+    {
+      "completion_length": 592.0,
+      "epoch": 18.48,
+      "grad_norm": 0.6363309621810913,
+      "kl": 0.770276665687561,
+      "learning_rate": 4.476723934359609e-06,
+      "loss": 0.0308,
+      "reward": 2.9551913738250732,
+      "reward_std": 0.18403679132461548,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.08019141852855682,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 462
+    },
+    {
+      "completion_length": 341.0,
+      "epoch": 18.52,
+      "grad_norm": 0.6210933327674866,
+      "kl": 0.530462384223938,
+      "learning_rate": 4.473380105482875e-06,
+      "loss": 0.0212,
+      "reward": 3.187347650527954,
+      "reward_std": 0.11080538481473923,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.1873476356267929,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 463
+    },
+    {
+      "completion_length": 483.0,
+      "epoch": 18.56,
+      "grad_norm": 0.5502752661705017,
+      "kl": 1.0583032369613647,
+      "learning_rate": 4.470026884016805e-06,
+      "loss": 0.0423,
+      "reward": 1.718454122543335,
+      "reward_std": 1.522723913192749,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.9065459370613098,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 464
+    },
+    {
+      "completion_length": 656.0,
+      "epoch": 18.6,
+      "grad_norm": 0.5941818952560425,
+      "kl": 1.023593544960022,
+      "learning_rate": 4.466664285921543e-06,
+      "loss": 0.0409,
+      "reward": -1.125,
+      "reward_std": 0.25,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 465
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 18.64,
+      "grad_norm": 0.5130367875099182,
+      "kl": 1.292258858680725,
+      "learning_rate": 4.463292327201862e-06,
+      "loss": 0.0517,
+      "reward": 2.46085524559021,
+      "reward_std": 0.42149004340171814,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.4141446650028229,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 466
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 18.68,
+      "grad_norm": 0.592551589012146,
+      "kl": 0.7206482291221619,
+      "learning_rate": 4.459911023907092e-06,
+      "loss": 0.0288,
+      "reward": 2.441016674041748,
+      "reward_std": 0.3532237708568573,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.06601664423942566,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 467
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 18.72,
+      "grad_norm": 0.8326399326324463,
+      "kl": 0.8357914090156555,
+      "learning_rate": 4.456520392131035e-06,
+      "loss": 0.0334,
+      "reward": 2.6499860286712646,
+      "reward_std": 0.7204391956329346,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.875,
+      "rewards/wrapped_driving_reward": 0.2749861180782318,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 468
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 18.76,
+      "grad_norm": 0.5404223799705505,
+      "kl": 0.6306906938552856,
+      "learning_rate": 4.453120448011897e-06,
+      "loss": 0.0252,
+      "reward": -1.2777777910232544,
+      "reward_std": 0.48432207107543945,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9722222089767456,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 469
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 18.8,
+      "grad_norm": 0.42292672395706177,
+      "kl": 1.1823399066925049,
+      "learning_rate": 4.4497112077322045e-06,
+      "loss": 0.0473,
+      "reward": -1.1458332538604736,
+      "reward_std": 0.23935678601264954,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9791666865348816,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 470
+    },
+    {
+      "completion_length": 413.0,
+      "epoch": 18.84,
+      "grad_norm": 0.5878785848617554,
+      "kl": 0.6619639992713928,
+      "learning_rate": 4.446292687518734e-06,
+      "loss": 0.0265,
+      "reward": 2.993729591369629,
+      "reward_std": 0.3032970428466797,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.006270239129662514,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 471
+    },
+    {
+      "completion_length": 522.0,
+      "epoch": 18.88,
+      "grad_norm": 0.616311252117157,
+      "kl": 1.3041799068450928,
+      "learning_rate": 4.442864903642428e-06,
+      "loss": 0.0522,
+      "reward": 2.797008514404297,
+      "reward_std": 0.7625215649604797,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9886363744735718,
+      "rewards/wrapped_driving_reward": 0.18337202072143555,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 472
+    },
+    {
+      "completion_length": 576.0,
+      "epoch": 18.92,
+      "grad_norm": 0.6430866122245789,
+      "kl": 0.9943831562995911,
+      "learning_rate": 4.439427872418321e-06,
+      "loss": 0.0398,
+      "reward": 3.5124568939208984,
+      "reward_std": 0.24802593886852264,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.5124570727348328,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 473
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 18.96,
+      "grad_norm": 0.42341163754463196,
+      "kl": 0.9567270278930664,
+      "learning_rate": 4.435981610205464e-06,
+      "loss": 0.0383,
+      "reward": 2.988504648208618,
+      "reward_std": 0.34725552797317505,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.1135048121213913,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 474
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 19.0,
+      "grad_norm": 0.5046776533126831,
+      "kl": 1.2646851539611816,
+      "learning_rate": 4.432526133406843e-06,
+      "loss": 0.0506,
+      "reward": 3.1114625930786133,
+      "reward_std": 0.835891842842102,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.6114627122879028,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 475
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 19.04,
+      "grad_norm": 0.4383629262447357,
+      "kl": 0.8644335269927979,
+      "learning_rate": 4.4290614584693005e-06,
+      "loss": 0.0346,
+      "reward": 2.6626029014587402,
+      "reward_std": 0.4813341796398163,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9642857313156128,
+      "rewards/wrapped_driving_reward": 0.07331724464893341,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 476
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 19.08,
+      "grad_norm": 0.46059325337409973,
+      "kl": 0.7996255159378052,
+      "learning_rate": 4.425587601883461e-06,
+      "loss": 0.032,
+      "reward": 1.2542250156402588,
+      "reward_std": 3.507305383682251,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7272727489471436,
+      "rewards/wrapped_driving_reward": -0.9730477929115295,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 477
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 19.12,
+      "grad_norm": 0.48959171772003174,
+      "kl": 2.0407848358154297,
+      "learning_rate": 4.422104580183649e-06,
+      "loss": 0.0816,
+      "reward": 3.022857189178467,
+      "reward_std": 0.16870427131652832,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.949999988079071,
+      "rewards/wrapped_driving_reward": 0.07285712659358978,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 478
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 19.16,
+      "grad_norm": 0.4211970567703247,
+      "kl": 0.9752914309501648,
+      "learning_rate": 4.418612409947814e-06,
+      "loss": 0.039,
+      "reward": 3.0421531200408936,
+      "reward_std": 0.3262219727039337,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8928571343421936,
+      "rewards/wrapped_driving_reward": 0.14929600059986115,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 479
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 19.2,
+      "grad_norm": 0.3921389877796173,
+      "kl": 0.8673704862594604,
+      "learning_rate": 4.415111107797445e-06,
+      "loss": 0.0347,
+      "reward": -1.6749999523162842,
+      "reward_std": 0.39475730061531067,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.824999988079071,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 480
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 19.24,
+      "grad_norm": 0.49681219458580017,
+      "kl": 1.1280070543289185,
+      "learning_rate": 4.4116006903975015e-06,
+      "loss": 0.0451,
+      "reward": 2.109313726425171,
+      "reward_std": 1.7568812370300293,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9045454263687134,
+      "rewards/wrapped_driving_reward": -0.5452316403388977,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 481
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 19.28,
+      "grad_norm": 0.5449213981628418,
+      "kl": 1.4810718297958374,
+      "learning_rate": 4.408081174456322e-06,
+      "loss": 0.0592,
+      "reward": 2.895340919494629,
+      "reward_std": 0.3929472267627716,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.14534106850624084,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 482
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 19.32,
+      "grad_norm": 0.46818554401397705,
+      "kl": 1.1236234903335571,
+      "learning_rate": 4.404552576725557e-06,
+      "loss": 0.0449,
+      "reward": -1.25,
+      "reward_std": 0.28867512941360474,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 483
+    },
+    {
+      "completion_length": 554.0,
+      "epoch": 19.36,
+      "grad_norm": 0.6208794116973877,
+      "kl": 1.3854421377182007,
+      "learning_rate": 4.401014914000078e-06,
+      "loss": 0.0554,
+      "reward": 3.256164073944092,
+      "reward_std": 0.4106229543685913,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.5061641335487366,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 484
+    },
+    {
+      "completion_length": 511.0,
+      "epoch": 19.4,
+      "grad_norm": 0.5070251822471619,
+      "kl": 1.0283787250518799,
+      "learning_rate": 4.397468203117905e-06,
+      "loss": 0.0411,
+      "reward": 3.588742256164551,
+      "reward_std": 0.2591555416584015,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9642857313156128,
+      "rewards/wrapped_driving_reward": 0.6244565844535828,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 485
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 19.44,
+      "grad_norm": 0.4414016604423523,
+      "kl": 1.3710557222366333,
+      "learning_rate": 4.393912460960125e-06,
+      "loss": 0.0548,
+      "reward": 2.9028897285461426,
+      "reward_std": 0.3639879524707794,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.09711016714572906,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 486
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 19.48,
+      "grad_norm": 0.8334219455718994,
+      "kl": 1.6127806901931763,
+      "learning_rate": 4.3903477044508066e-06,
+      "loss": 0.0645,
+      "reward": -1.5,
+      "reward_std": 0.40824830532073975,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 487
+    },
+    {
+      "completion_length": 656.0,
+      "epoch": 19.52,
+      "grad_norm": 0.4386281371116638,
+      "kl": 0.916235089302063,
+      "learning_rate": 4.386773950556931e-06,
+      "loss": 0.0366,
+      "reward": 2.9088566303253174,
+      "reward_std": 0.24869580566883087,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9545454382896423,
+      "rewards/wrapped_driving_reward": -0.045688893646001816,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 488
+    },
+    {
+      "completion_length": 741.0,
+      "epoch": 19.56,
+      "grad_norm": 0.4149533212184906,
+      "kl": 0.9770787358283997,
+      "learning_rate": 4.3831912162882946e-06,
+      "loss": 0.0391,
+      "reward": 1.848495602607727,
+      "reward_std": 2.5709009170532227,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.901504397392273,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 489
+    },
+    {
+      "completion_length": 587.0,
+      "epoch": 19.6,
+      "grad_norm": 0.4931059181690216,
+      "kl": 0.6863958835601807,
+      "learning_rate": 4.379599518697444e-06,
+      "loss": 0.0275,
+      "reward": 1.1175979375839233,
+      "reward_std": 2.750753402709961,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.2574020624160767,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 490
+    },
+    {
+      "completion_length": 543.0,
+      "epoch": 19.64,
+      "grad_norm": 0.5057757496833801,
+      "kl": 1.1397721767425537,
+      "learning_rate": 4.375998874879585e-06,
+      "loss": 0.0456,
+      "reward": 3.4474494457244873,
+      "reward_std": 0.37866705656051636,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.4474495053291321,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 491
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 19.68,
+      "grad_norm": 0.46242570877075195,
+      "kl": 1.0390926599502563,
+      "learning_rate": 4.372389301972506e-06,
+      "loss": 0.0416,
+      "reward": 1.491578459739685,
+      "reward_std": 3.673814058303833,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.7584214806556702,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 492
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 19.72,
+      "grad_norm": 0.38337427377700806,
+      "kl": 0.9781420230865479,
+      "learning_rate": 4.368770817156493e-06,
+      "loss": 0.0391,
+      "reward": 1.3044135570526123,
+      "reward_std": 2.237086534500122,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9722222089767456,
+      "rewards/wrapped_driving_reward": -1.2928086519241333,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 493
+    },
+    {
+      "completion_length": 528.0,
+      "epoch": 19.76,
+      "grad_norm": 0.5733584761619568,
+      "kl": 0.773429811000824,
+      "learning_rate": 4.365143437654249e-06,
+      "loss": 0.0309,
+      "reward": 3.358289957046509,
+      "reward_std": 0.4428289532661438,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.35829001665115356,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 494
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 19.8,
+      "grad_norm": 0.38473784923553467,
+      "kl": 0.9460413455963135,
+      "learning_rate": 4.3615071807308165e-06,
+      "loss": 0.0378,
+      "reward": 2.6372628211975098,
+      "reward_std": 0.507134735584259,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9027777910232544,
+      "rewards/wrapped_driving_reward": -0.015514791011810303,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 495
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 19.84,
+      "grad_norm": 0.5427440404891968,
+      "kl": 1.3654718399047852,
+      "learning_rate": 4.357862063693486e-06,
+      "loss": 0.0546,
+      "reward": -1.053030252456665,
+      "reward_std": 0.07872962206602097,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9469696879386902,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 496
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 19.88,
+      "grad_norm": 0.6441575884819031,
+      "kl": 1.5351756811141968,
+      "learning_rate": 4.354208103891723e-06,
+      "loss": 0.0614,
+      "reward": 2.9220848083496094,
+      "reward_std": 0.5418745875358582,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9583333134651184,
+      "rewards/wrapped_driving_reward": 0.46375155448913574,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 497
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 19.92,
+      "grad_norm": 0.3897554874420166,
+      "kl": 0.6224919557571411,
+      "learning_rate": 4.350545318717081e-06,
+      "loss": 0.0249,
+      "reward": 0.7557200193405151,
+      "reward_std": 2.027337074279785,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -2.1192798614501953,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 498
+    },
+    {
+      "completion_length": 539.0,
+      "epoch": 19.96,
+      "grad_norm": 0.501442015171051,
+      "kl": 0.8325724601745605,
+      "learning_rate": 4.3468737256031155e-06,
+      "loss": 0.0333,
+      "reward": -1.125,
+      "reward_std": 0.25,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 499
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 20.0,
+      "grad_norm": 0.3670285940170288,
+      "kl": 1.4814090728759766,
+      "learning_rate": 4.34319334202531e-06,
+      "loss": 0.0593,
+      "reward": 2.8780927658081055,
+      "reward_std": 0.0776260644197464,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9750000238418579,
+      "rewards/wrapped_driving_reward": -0.0969071239233017,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 500
+    },
+    {
+      "completion_length": 720.0,
+      "epoch": 20.04,
+      "grad_norm": 0.5521990060806274,
+      "kl": 1.595299243927002,
+      "learning_rate": 4.339504185500984e-06,
+      "loss": 0.0638,
+      "reward": 1.5800509452819824,
+      "reward_std": 3.053708791732788,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9199489951133728,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 501
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 20.08,
+      "grad_norm": 0.45915642380714417,
+      "kl": 1.1580920219421387,
+      "learning_rate": 4.335806273589214e-06,
+      "loss": 0.0463,
+      "reward": -1.4583332538604736,
+      "reward_std": 0.5335936546325684,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9166666865348816,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 502
+    },
+    {
+      "completion_length": 515.0,
+      "epoch": 20.12,
+      "grad_norm": 0.5437774062156677,
+      "kl": 1.1531405448913574,
+      "learning_rate": 4.332099623890749e-06,
+      "loss": 0.0461,
+      "reward": 3.0214743614196777,
+      "reward_std": 0.1465667486190796,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.96875,
+      "rewards/wrapped_driving_reward": 0.052724581211805344,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 503
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 20.16,
+      "grad_norm": 0.661466121673584,
+      "kl": 1.5797182321548462,
+      "learning_rate": 4.328384254047927e-06,
+      "loss": 0.0632,
+      "reward": 3.216904640197754,
+      "reward_std": 0.47574853897094727,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.935606062412262,
+      "rewards/wrapped_driving_reward": 0.2812984585762024,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 504
+    },
+    {
+      "completion_length": 619.0,
+      "epoch": 20.2,
+      "grad_norm": 0.535003662109375,
+      "kl": 1.1837904453277588,
+      "learning_rate": 4.324660181744589e-06,
+      "loss": 0.0474,
+      "reward": 2.576167345046997,
+      "reward_std": 0.9729455709457397,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.4238327443599701,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 505
+    },
+    {
+      "completion_length": 679.0,
+      "epoch": 20.24,
+      "grad_norm": 0.6109884977340698,
+      "kl": 0.4138145446777344,
+      "learning_rate": 4.320927424706001e-06,
+      "loss": 0.0166,
+      "reward": 2.9425160884857178,
+      "reward_std": 0.9789575338363647,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8333333730697632,
+      "rewards/wrapped_driving_reward": 0.35918280482292175,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 506
+    },
+    {
+      "completion_length": 483.0,
+      "epoch": 20.28,
+      "grad_norm": 0.4926319122314453,
+      "kl": 0.7010176777839661,
+      "learning_rate": 4.317186000698761e-06,
+      "loss": 0.028,
+      "reward": 2.29917049407959,
+      "reward_std": 0.5825653076171875,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9772727489471436,
+      "rewards/wrapped_driving_reward": -0.6781020760536194,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 507
+    },
+    {
+      "completion_length": 696.0,
+      "epoch": 20.32,
+      "grad_norm": 0.3954756557941437,
+      "kl": 0.993597686290741,
+      "learning_rate": 4.313435927530719e-06,
+      "loss": 0.0397,
+      "reward": -1.375,
+      "reward_std": 0.4787135720252991,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 508
+    },
+    {
+      "completion_length": 381.0,
+      "epoch": 20.36,
+      "grad_norm": 0.5946969985961914,
+      "kl": 0.3980226218700409,
+      "learning_rate": 4.309677223050895e-06,
+      "loss": 0.0159,
+      "reward": 1.2032806873321533,
+      "reward_std": 2.8211827278137207,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.1717194318771362,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 509
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 20.4,
+      "grad_norm": 0.4983665943145752,
+      "kl": 0.5070443749427795,
+      "learning_rate": 4.305909905149389e-06,
+      "loss": 0.0203,
+      "reward": -1.625,
+      "reward_std": 0.4787135720252991,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 510
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 20.44,
+      "grad_norm": 0.43326103687286377,
+      "kl": 1.5419466495513916,
+      "learning_rate": 4.3021339917572975e-06,
+      "loss": 0.0617,
+      "reward": 2.3160929679870605,
+      "reward_std": 0.9564554691314697,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.5589069128036499,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 511
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 20.48,
+      "grad_norm": 0.5289414525032043,
+      "kl": 1.1936659812927246,
+      "learning_rate": 4.2983495008466285e-06,
+      "loss": 0.0477,
+      "reward": 0.6631726026535034,
+      "reward_std": 1.2805912494659424,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -2.086827278137207,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 512
+    },
+    {
+      "completion_length": 630.0,
+      "epoch": 20.52,
+      "grad_norm": 0.7556226253509521,
+      "kl": 1.03369140625,
+      "learning_rate": 4.294556450430216e-06,
+      "loss": 0.0413,
+      "reward": 2.696643352508545,
+      "reward_std": 0.5111956596374512,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9642857313156128,
+      "rewards/wrapped_driving_reward": -0.2676423490047455,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 513
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 20.56,
+      "grad_norm": 0.6190569996833801,
+      "kl": 0.35693028569221497,
+      "learning_rate": 4.290754858561636e-06,
+      "loss": 0.0143,
+      "reward": 2.3937301635742188,
+      "reward_std": 0.33194005489349365,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.143730029463768,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 514
+    },
+    {
+      "completion_length": 428.0,
+      "epoch": 20.6,
+      "grad_norm": 0.7335030436515808,
+      "kl": 0.48697853088378906,
+      "learning_rate": 4.2869447433351165e-06,
+      "loss": 0.0195,
+      "reward": 3.3049356937408447,
+      "reward_std": 0.6103650331497192,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.4299355745315552,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 515
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 20.64,
+      "grad_norm": 0.5130233764648438,
+      "kl": 1.0960686206817627,
+      "learning_rate": 4.283126122885455e-06,
+      "loss": 0.0438,
+      "reward": 2.9208106994628906,
+      "reward_std": 0.15237730741500854,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.17081058025360107,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 516
+    },
+    {
+      "completion_length": 682.0,
+      "epoch": 20.68,
+      "grad_norm": 1.5805164575576782,
+      "kl": 1.2938767671585083,
+      "learning_rate": 4.2792990153879286e-06,
+      "loss": 0.0518,
+      "reward": 2.5840139389038086,
+      "reward_std": 0.5336389541625977,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.08401414752006531,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 517
+    },
+    {
+      "completion_length": 518.0,
+      "epoch": 20.72,
+      "grad_norm": 0.795953094959259,
+      "kl": 1.048414945602417,
+      "learning_rate": 4.275463439058214e-06,
+      "loss": 0.0419,
+      "reward": 3.040872097015381,
+      "reward_std": 0.6307140588760376,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8392857313156128,
+      "rewards/wrapped_driving_reward": 0.20158648490905762,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 518
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 20.76,
+      "grad_norm": 1.237001657485962,
+      "kl": 1.0653915405273438,
+      "learning_rate": 4.271619412152293e-06,
+      "loss": 0.0426,
+      "reward": 2.947390079498291,
+      "reward_std": 0.2647410035133362,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9166666865348816,
+      "rewards/wrapped_driving_reward": 0.03072343021631241,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 519
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 20.8,
+      "grad_norm": 2.3102457523345947,
+      "kl": 1.6519412994384766,
+      "learning_rate": 4.267766952966369e-06,
+      "loss": 0.0661,
+      "reward": 2.849301338195801,
+      "reward_std": 0.4467388987541199,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.09930121898651123,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 520
+    },
+    {
+      "completion_length": 450.0,
+      "epoch": 20.84,
+      "grad_norm": 0.6221670508384705,
+      "kl": 0.32503193616867065,
+      "learning_rate": 4.2639060798367835e-06,
+      "loss": 0.013,
+      "reward": -1.125,
+      "reward_std": 0.25,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 521
+    },
+    {
+      "completion_length": 690.0,
+      "epoch": 20.88,
+      "grad_norm": 0.8544734120368958,
+      "kl": 0.9178805351257324,
+      "learning_rate": 4.260036811139922e-06,
+      "loss": 0.0367,
+      "reward": 1.6469182968139648,
+      "reward_std": 3.2096569538116455,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.6030816435813904,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 522
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 20.92,
+      "grad_norm": 1.0491329431533813,
+      "kl": 1.2910516262054443,
+      "learning_rate": 4.25615916529213e-06,
+      "loss": 0.0516,
+      "reward": 2.478778839111328,
+      "reward_std": 0.7036353945732117,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9166666865348816,
+      "rewards/wrapped_driving_reward": -0.1878880113363266,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 523
+    },
+    {
+      "completion_length": 639.0,
+      "epoch": 20.96,
+      "grad_norm": 0.7800697088241577,
+      "kl": 0.911841869354248,
+      "learning_rate": 4.2522731607496275e-06,
+      "loss": 0.0365,
+      "reward": 3.115720272064209,
+      "reward_std": 0.08318884670734406,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.11572031676769257,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 524
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 21.0,
+      "grad_norm": 0.4819512367248535,
+      "kl": 0.9187918901443481,
+      "learning_rate": 4.248378816008418e-06,
+      "loss": 0.0368,
+      "reward": -1.0,
+      "reward_std": 0.0,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 525
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 21.04,
+      "grad_norm": 1.4163870811462402,
+      "kl": 1.6186529397964478,
+      "learning_rate": 4.244476149604201e-06,
+      "loss": 0.0647,
+      "reward": 2.948265552520752,
+      "reward_std": 0.6075405478477478,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9545454382896423,
+      "rewards/wrapped_driving_reward": 0.11871998757123947,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 526
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 21.08,
+      "grad_norm": 0.9614571332931519,
+      "kl": 0.9680662751197815,
+      "learning_rate": 4.2405651801122835e-06,
+      "loss": 0.0387,
+      "reward": -1.6165865659713745,
+      "reward_std": 0.4794272482395172,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8834134340286255,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 527
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 21.12,
+      "grad_norm": 0.7541413307189941,
+      "kl": 1.3542858362197876,
+      "learning_rate": 4.236645926147493e-06,
+      "loss": 0.0542,
+      "reward": 2.6069326400756836,
+      "reward_std": 0.21937526762485504,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9791666865348816,
+      "rewards/wrapped_driving_reward": -0.372234046459198,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 528
+    },
+    {
+      "completion_length": 512.0,
+      "epoch": 21.16,
+      "grad_norm": 0.5188172459602356,
+      "kl": 0.6754733920097351,
+      "learning_rate": 4.2327184063640905e-06,
+      "loss": 0.027,
+      "reward": 3.0157723426818848,
+      "reward_std": 0.20528268814086914,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.875,
+      "rewards/wrapped_driving_reward": 0.14077217876911163,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 529
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 21.2,
+      "grad_norm": 0.5220324397087097,
+      "kl": 1.3572345972061157,
+      "learning_rate": 4.228782639455674e-06,
+      "loss": 0.0543,
+      "reward": 2.6850199699401855,
+      "reward_std": 0.41600048542022705,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.06001979485154152,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 530
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 21.24,
+      "grad_norm": 0.5855657458305359,
+      "kl": 0.9850445985794067,
+      "learning_rate": 4.224838644155099e-06,
+      "loss": 0.0394,
+      "reward": 0.7505922317504883,
+      "reward_std": 3.3031928539276123,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.4994077682495117,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 531
+    },
+    {
+      "completion_length": 560.0,
+      "epoch": 21.28,
+      "grad_norm": 0.5909777283668518,
+      "kl": 1.309668779373169,
+      "learning_rate": 4.220886439234385e-06,
+      "loss": 0.0524,
+      "reward": 1.8188085556030273,
+      "reward_std": 3.256636619567871,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.43119144439697266,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 532
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 21.32,
+      "grad_norm": 0.38810762763023376,
+      "kl": 1.0575664043426514,
+      "learning_rate": 4.216926043504626e-06,
+      "loss": 0.0423,
+      "reward": 2.893254041671753,
+      "reward_std": 0.7650725245475769,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.2682541012763977,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 533
+    },
+    {
+      "completion_length": 711.0,
+      "epoch": 21.36,
+      "grad_norm": 0.560529887676239,
+      "kl": 1.4421268701553345,
+      "learning_rate": 4.212957475815898e-06,
+      "loss": 0.0577,
+      "reward": 3.511518955230713,
+      "reward_std": 0.41801729798316956,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9375,
+      "rewards/wrapped_driving_reward": 0.5740189552307129,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 534
+    },
+    {
+      "completion_length": 486.0,
+      "epoch": 21.4,
+      "grad_norm": 0.9958294630050659,
+      "kl": 0.6733448505401611,
+      "learning_rate": 4.2089807550571786e-06,
+      "loss": 0.0269,
+      "reward": -1.25,
+      "reward_std": 0.28867512941360474,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 535
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 21.44,
+      "grad_norm": 0.5459821820259094,
+      "kl": 1.3347562551498413,
+      "learning_rate": 4.204995900156247e-06,
+      "loss": 0.0534,
+      "reward": 3.5074427127838135,
+      "reward_std": 0.6772680878639221,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.5074427127838135,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 536
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 21.48,
+      "grad_norm": 0.5361037850379944,
+      "kl": 1.1742392778396606,
+      "learning_rate": 4.2010029300795986e-06,
+      "loss": 0.047,
+      "reward": -1.625,
+      "reward_std": 1.25,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 537
+    },
+    {
+      "completion_length": 516.0,
+      "epoch": 21.52,
+      "grad_norm": 12.987887382507324,
+      "kl": 2.4212050437927246,
+      "learning_rate": 4.197001863832355e-06,
+      "loss": 0.0968,
+      "reward": 2.888948678970337,
+      "reward_std": 0.5455219745635986,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9285714626312256,
+      "rewards/wrapped_driving_reward": -0.039622798562049866,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 538
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 21.56,
+      "grad_norm": 0.40430188179016113,
+      "kl": 0.3120957911014557,
+      "learning_rate": 4.192992720458172e-06,
+      "loss": 0.0125,
+      "reward": 2.160168170928955,
+      "reward_std": 0.5079329013824463,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.21483179926872253,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 539
+    },
+    {
+      "completion_length": 735.0,
+      "epoch": 21.6,
+      "grad_norm": 0.48742803931236267,
+      "kl": 1.1038862466812134,
+      "learning_rate": 4.188975519039151e-06,
+      "loss": 0.0442,
+      "reward": 2.9028220176696777,
+      "reward_std": 0.44946083426475525,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.949999988079071,
+      "rewards/wrapped_driving_reward": 0.20282205939292908,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 540
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 21.64,
+      "grad_norm": 0.40997225046157837,
+      "kl": 1.483988642692566,
+      "learning_rate": 4.184950278695745e-06,
+      "loss": 0.0594,
+      "reward": 3.176731586456299,
+      "reward_std": 1.1836580038070679,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.426731675863266,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 541
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 21.68,
+      "grad_norm": 0.6240445375442505,
+      "kl": 1.3704639673233032,
+      "learning_rate": 4.18091701858667e-06,
+      "loss": 0.0548,
+      "reward": 1.4026062488555908,
+      "reward_std": 3.274519920349121,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.7223937511444092,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 542
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 21.72,
+      "grad_norm": 0.4963267147541046,
+      "kl": 1.1881283521652222,
+      "learning_rate": 4.1768757579088145e-06,
+      "loss": 0.0475,
+      "reward": 2.731823682785034,
+      "reward_std": 0.5495507717132568,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.1068236380815506,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 543
+    },
+    {
+      "completion_length": 566.0,
+      "epoch": 21.76,
+      "grad_norm": 0.6541410088539124,
+      "kl": 1.4289416074752808,
+      "learning_rate": 4.172826515897146e-06,
+      "loss": 0.0572,
+      "reward": -1.1666667461395264,
+      "reward_std": 0.3333333730697632,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9583333134651184,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 544
+    },
+    {
+      "completion_length": 540.0,
+      "epoch": 21.8,
+      "grad_norm": 0.46453943848609924,
+      "kl": 0.6854754686355591,
+      "learning_rate": 4.168769311824619e-06,
+      "loss": 0.0274,
+      "reward": 3.0061306953430176,
+      "reward_std": 0.44360265135765076,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9772727489471436,
+      "rewards/wrapped_driving_reward": 0.028858043253421783,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 545
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 21.84,
+      "grad_norm": 0.42537233233451843,
+      "kl": 1.2528382539749146,
+      "learning_rate": 4.164704165002086e-06,
+      "loss": 0.0501,
+      "reward": 2.6121349334716797,
+      "reward_std": 0.5790495276451111,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.1378651112318039,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 546
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 21.88,
+      "grad_norm": 0.42918747663497925,
+      "kl": 1.3132213354110718,
+      "learning_rate": 4.160631094778205e-06,
+      "loss": 0.0525,
+      "reward": -1.25,
+      "reward_std": 0.5,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 547
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 21.92,
+      "grad_norm": 0.6208239793777466,
+      "kl": 1.4367201328277588,
+      "learning_rate": 4.1565501205393445e-06,
+      "loss": 0.0575,
+      "reward": 2.702597141265869,
+      "reward_std": 0.7576584219932556,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.17240279912948608,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 548
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 21.96,
+      "grad_norm": 0.514549732208252,
+      "kl": 1.6398608684539795,
+      "learning_rate": 4.152461261709494e-06,
+      "loss": 0.0656,
+      "reward": 2.376189947128296,
+      "reward_std": 0.6441675424575806,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.2488100230693817,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 549
+    },
+    {
+      "completion_length": 451.0,
+      "epoch": 22.0,
+      "grad_norm": 0.5030242800712585,
+      "kl": 1.0479674339294434,
+      "learning_rate": 4.1483645377501726e-06,
+      "loss": 0.0419,
+      "reward": 3.0598669052124023,
+      "reward_std": 0.3327578008174896,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.18486672639846802,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 550
+    },
+    {
+      "completion_length": 622.0,
+      "epoch": 22.04,
+      "grad_norm": 0.38805630803108215,
+      "kl": 0.7481611371040344,
+      "learning_rate": 4.144259968160332e-06,
+      "loss": 0.0299,
+      "reward": 3.129121780395508,
+      "reward_std": 0.25338175892829895,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.12912192940711975,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 551
+    },
+    {
+      "completion_length": 494.0,
+      "epoch": 22.08,
+      "grad_norm": 0.550291895866394,
+      "kl": 1.145378828048706,
+      "learning_rate": 4.140147572476269e-06,
+      "loss": 0.0458,
+      "reward": 2.7440075874328613,
+      "reward_std": 0.8928554654121399,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.25599223375320435,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 552
+    },
+    {
+      "completion_length": 542.0,
+      "epoch": 22.12,
+      "grad_norm": 0.47073522210121155,
+      "kl": 1.0155128240585327,
+      "learning_rate": 4.136027370271526e-06,
+      "loss": 0.0406,
+      "reward": 1.1688520908355713,
+      "reward_std": 2.8173623085021973,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.6041666865348816,
+      "rewards/wrapped_driving_reward": -0.9353145360946655,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 553
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 22.16,
+      "grad_norm": 0.7193614840507507,
+      "kl": 1.4088541269302368,
+      "learning_rate": 4.1318993811568065e-06,
+      "loss": 0.0564,
+      "reward": 3.4254112243652344,
+      "reward_std": 0.43707841634750366,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.4254113435745239,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 554
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 22.2,
+      "grad_norm": 0.3912142515182495,
+      "kl": 1.4723032712936401,
+      "learning_rate": 4.127763624779873e-06,
+      "loss": 0.0589,
+      "reward": 0.24002844095230103,
+      "reward_std": 1.8333266973495483,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -2.6349716186523438,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 555
+    },
+    {
+      "completion_length": 612.0,
+      "epoch": 22.24,
+      "grad_norm": 0.8369085192680359,
+      "kl": 1.3998736143112183,
+      "learning_rate": 4.123620120825459e-06,
+      "loss": 0.056,
+      "reward": 2.403351306915283,
+      "reward_std": 0.4577612578868866,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9583333134651184,
+      "rewards/wrapped_driving_reward": -0.054981887340545654,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 556
+    },
+    {
+      "completion_length": 477.0,
+      "epoch": 22.28,
+      "grad_norm": 0.5052926540374756,
+      "kl": 0.6547082662582397,
+      "learning_rate": 4.119468889015175e-06,
+      "loss": 0.0262,
+      "reward": 1.30754554271698,
+      "reward_std": 2.665022611618042,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.69245445728302,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 557
+    },
+    {
+      "completion_length": 739.0,
+      "epoch": 22.32,
+      "grad_norm": 0.6290760636329651,
+      "kl": 1.5631662607192993,
+      "learning_rate": 4.11530994910741e-06,
+      "loss": 0.0625,
+      "reward": -1.0499999523162842,
+      "reward_std": 0.10000002384185791,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.949999988079071,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 558
+    },
+    {
+      "completion_length": 364.0,
+      "epoch": 22.36,
+      "grad_norm": 0.6056944727897644,
+      "kl": 0.5589709877967834,
+      "learning_rate": 4.111143320897244e-06,
+      "loss": 0.0224,
+      "reward": 3.2965807914733887,
+      "reward_std": 0.5819244384765625,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.2965807318687439,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 559
+    },
+    {
+      "completion_length": 682.0,
+      "epoch": 22.4,
+      "grad_norm": 0.4512479305267334,
+      "kl": 1.2240394353866577,
+      "learning_rate": 4.106969024216348e-06,
+      "loss": 0.049,
+      "reward": 2.993767738342285,
+      "reward_std": 0.24015867710113525,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.006232157349586487,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 560
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 22.44,
+      "grad_norm": 0.4886869490146637,
+      "kl": 1.3827241659164429,
+      "learning_rate": 4.102787078932896e-06,
+      "loss": 0.0553,
+      "reward": 2.665478229522705,
+      "reward_std": 0.4790554344654083,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.08452148735523224,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 561
+    },
+    {
+      "completion_length": 673.0,
+      "epoch": 22.48,
+      "grad_norm": 0.3894011080265045,
+      "kl": 1.2513083219528198,
+      "learning_rate": 4.098597504951462e-06,
+      "loss": 0.0501,
+      "reward": -1.625,
+      "reward_std": 0.4787135720252991,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 562
+    },
+    {
+      "completion_length": 631.0,
+      "epoch": 22.52,
+      "grad_norm": 0.4578774869441986,
+      "kl": 1.1499855518341064,
+      "learning_rate": 4.094400322212933e-06,
+      "loss": 0.046,
+      "reward": 2.9452905654907227,
+      "reward_std": 0.7577587366104126,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.05470933020114899,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 563
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 22.56,
+      "grad_norm": 0.423875093460083,
+      "kl": 0.8647013902664185,
+      "learning_rate": 4.09019555069441e-06,
+      "loss": 0.0346,
+      "reward": 2.480205535888672,
+      "reward_std": 0.4056702256202698,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.10520540177822113,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 564
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 22.6,
+      "grad_norm": 0.39600902795791626,
+      "kl": 1.215203881263733,
+      "learning_rate": 4.085983210409114e-06,
+      "loss": 0.0486,
+      "reward": 2.527067184448242,
+      "reward_std": 0.6684492826461792,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8928571343421936,
+      "rewards/wrapped_driving_reward": 0.13421006500720978,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 565
+    },
+    {
+      "completion_length": 492.0,
+      "epoch": 22.64,
+      "grad_norm": 4.348442554473877,
+      "kl": 0.9590518474578857,
+      "learning_rate": 4.081763321406291e-06,
+      "loss": 0.0384,
+      "reward": -1.0499999523162842,
+      "reward_std": 0.10000002384185791,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.949999988079071,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 566
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 22.68,
+      "grad_norm": 0.5797974467277527,
+      "kl": 1.4900603294372559,
+      "learning_rate": 4.077535903771115e-06,
+      "loss": 0.0596,
+      "reward": 3.0958666801452637,
+      "reward_std": 0.3664493262767792,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.22086681425571442,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 567
+    },
+    {
+      "completion_length": 600.0,
+      "epoch": 22.72,
+      "grad_norm": 0.5184460878372192,
+      "kl": 1.305143117904663,
+      "learning_rate": 4.073300977624594e-06,
+      "loss": 0.0522,
+      "reward": 2.9294841289520264,
+      "reward_std": 0.5147085189819336,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8958333134651184,
+      "rewards/wrapped_driving_reward": 0.28365081548690796,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 568
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 22.76,
+      "grad_norm": 0.9449083209037781,
+      "kl": 1.0648530721664429,
+      "learning_rate": 4.069058563123476e-06,
+      "loss": 0.0426,
+      "reward": -1.25,
+      "reward_std": 0.5,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 569
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 22.8,
+      "grad_norm": 0.6300418376922607,
+      "kl": 1.4819085597991943,
+      "learning_rate": 4.064808680460149e-06,
+      "loss": 0.0593,
+      "reward": 2.6562626361846924,
+      "reward_std": 0.36848315596580505,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.21873745322227478,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 570
+    },
+    {
+      "completion_length": 681.0,
+      "epoch": 22.84,
+      "grad_norm": 0.45206159353256226,
+      "kl": 0.7465510368347168,
+      "learning_rate": 4.060551349862545e-06,
+      "loss": 0.0299,
+      "reward": 2.4971559047698975,
+      "reward_std": 0.5323460102081299,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.3778441548347473,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 571
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 22.88,
+      "grad_norm": 0.49330395460128784,
+      "kl": 1.0091811418533325,
+      "learning_rate": 4.056286591594049e-06,
+      "loss": 0.0404,
+      "reward": 1.2864571809768677,
+      "reward_std": 2.8978612422943115,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9635427594184875,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 572
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 22.92,
+      "grad_norm": 1.236677885055542,
+      "kl": 1.20204758644104,
+      "learning_rate": 4.052014425953399e-06,
+      "loss": 0.0481,
+      "reward": -1.2884615659713745,
+      "reward_std": 0.4798709750175476,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9615384340286255,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 573
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 22.96,
+      "grad_norm": 0.36019617319107056,
+      "kl": 1.6590219736099243,
+      "learning_rate": 4.047734873274586e-06,
+      "loss": 0.0664,
+      "reward": 3.00655460357666,
+      "reward_std": 0.28616681694984436,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.13155463337898254,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 574
+    },
+    {
+      "completion_length": 577.0,
+      "epoch": 23.0,
+      "grad_norm": 1.064710259437561,
+      "kl": 1.6679836511611938,
+      "learning_rate": 4.043447953926763e-06,
+      "loss": 0.0667,
+      "reward": 3.2319164276123047,
+      "reward_std": 0.5175439119338989,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.23191656172275543,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 575
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 23.04,
+      "grad_norm": 0.46895185112953186,
+      "kl": 1.807165265083313,
+      "learning_rate": 4.039153688314146e-06,
+      "loss": 0.0723,
+      "reward": -1.0833332538604736,
+      "reward_std": 0.16666662693023682,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9166666865348816,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 576
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 23.08,
+      "grad_norm": 0.3290964365005493,
+      "kl": 1.4155315160751343,
+      "learning_rate": 4.034852096875917e-06,
+      "loss": 0.0566,
+      "reward": 1.61959969997406,
+      "reward_std": 2.1258249282836914,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.00540030002594,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 577
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 23.12,
+      "grad_norm": 0.5254642367362976,
+      "kl": 0.9085728526115417,
+      "learning_rate": 4.0305432000861236e-06,
+      "loss": 0.0363,
+      "reward": 2.5806849002838135,
+      "reward_std": 0.534013032913208,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9375,
+      "rewards/wrapped_driving_reward": 0.018184844404459,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 578
+    },
+    {
+      "completion_length": 485.0,
+      "epoch": 23.16,
+      "grad_norm": 0.5562568306922913,
+      "kl": 1.061842918395996,
+      "learning_rate": 4.026227018453587e-06,
+      "loss": 0.0425,
+      "reward": 1.7475911378860474,
+      "reward_std": 0.9556852579116821,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.0024088621139526,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 579
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 23.2,
+      "grad_norm": 0.7456584572792053,
+      "kl": 1.79913330078125,
+      "learning_rate": 4.021903572521802e-06,
+      "loss": 0.072,
+      "reward": 3.1010560989379883,
+      "reward_std": 0.11367816478013992,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.10105594992637634,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 580
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 23.24,
+      "grad_norm": 0.3952648341655731,
+      "kl": 1.7831506729125977,
+      "learning_rate": 4.0175728828688355e-06,
+      "loss": 0.0713,
+      "reward": 3.3662846088409424,
+      "reward_std": 0.44146811962127686,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9565972089767456,
+      "rewards/wrapped_driving_reward": 0.40968745946884155,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 581
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 23.28,
+      "grad_norm": 0.5361789464950562,
+      "kl": 1.3672832250595093,
+      "learning_rate": 4.013234970107236e-06,
+      "loss": 0.0547,
+      "reward": 2.194763422012329,
+      "reward_std": 0.9637414216995239,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.8052366375923157,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 582
+    },
+    {
+      "completion_length": 559.0,
+      "epoch": 23.32,
+      "grad_norm": 0.5380634069442749,
+      "kl": 1.4674052000045776,
+      "learning_rate": 4.0088898548839285e-06,
+      "loss": 0.0587,
+      "reward": 2.7343053817749023,
+      "reward_std": 0.47148850560188293,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.10930530726909637,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 583
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 23.36,
+      "grad_norm": 0.4427616000175476,
+      "kl": 0.7964285612106323,
+      "learning_rate": 4.0045375578801216e-06,
+      "loss": 0.0319,
+      "reward": 1.0508185625076294,
+      "reward_std": 3.381551742553711,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7142857313156128,
+      "rewards/wrapped_driving_reward": -1.0384670495986938,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 584
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 23.4,
+      "grad_norm": 0.4768621623516083,
+      "kl": 0.8902795910835266,
+      "learning_rate": 4.000178099811203e-06,
+      "loss": 0.0356,
+      "reward": 0.9182654619216919,
+      "reward_std": 2.618046760559082,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9567345976829529,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 585
+    },
+    {
+      "completion_length": 561.0,
+      "epoch": 23.44,
+      "grad_norm": 0.4203995168209076,
+      "kl": 0.7094324231147766,
+      "learning_rate": 3.995811501426648e-06,
+      "loss": 0.0284,
+      "reward": 2.076686143875122,
+      "reward_std": 2.1102030277252197,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.9233137965202332,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 586
+    },
+    {
+      "completion_length": 655.0,
+      "epoch": 23.48,
+      "grad_norm": 0.6416264772415161,
+      "kl": 1.6330536603927612,
+      "learning_rate": 3.991437783509916e-06,
+      "loss": 0.0653,
+      "reward": 3.16111421585083,
+      "reward_std": 0.5641868710517883,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9375,
+      "rewards/wrapped_driving_reward": 0.3486141860485077,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 587
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 23.52,
+      "grad_norm": 0.5578856468200684,
+      "kl": 1.2766082286834717,
+      "learning_rate": 3.987056966878354e-06,
+      "loss": 0.0511,
+      "reward": -1.5277777910232544,
+      "reward_std": 0.3643020987510681,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9722222089767456,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 588
+    },
+    {
+      "completion_length": 580.0,
+      "epoch": 23.56,
+      "grad_norm": 0.6553155779838562,
+      "kl": 1.447812795639038,
+      "learning_rate": 3.982669072383093e-06,
+      "loss": 0.0579,
+      "reward": -1.375,
+      "reward_std": 0.4787135720252991,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 589
+    },
+    {
+      "completion_length": 472.0,
+      "epoch": 23.6,
+      "grad_norm": 0.515069842338562,
+      "kl": 0.9640145897865295,
+      "learning_rate": 3.978274120908957e-06,
+      "loss": 0.0386,
+      "reward": -1.21875,
+      "reward_std": 0.21347814798355103,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.90625,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 590
+    },
+    {
+      "completion_length": 610.0,
+      "epoch": 23.64,
+      "grad_norm": 0.4059649109840393,
+      "kl": 1.3692982196807861,
+      "learning_rate": 3.973872133374354e-06,
+      "loss": 0.0548,
+      "reward": 3.1105618476867676,
+      "reward_std": 0.07002107799053192,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9270833134651184,
+      "rewards/wrapped_driving_reward": 0.18347838521003723,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 591
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 23.68,
+      "grad_norm": 0.7165226936340332,
+      "kl": 1.6977932453155518,
+      "learning_rate": 3.969463130731183e-06,
+      "loss": 0.0679,
+      "reward": 2.785404682159424,
+      "reward_std": 0.4484281837940216,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.03540457785129547,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 592
+    },
+    {
+      "completion_length": 510.0,
+      "epoch": 23.72,
+      "grad_norm": 0.5600489377975464,
+      "kl": 0.7112762928009033,
+      "learning_rate": 3.965047133964735e-06,
+      "loss": 0.0285,
+      "reward": 2.8892576694488525,
+      "reward_std": 0.4215315580368042,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.014257688075304031,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 593
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 23.76,
+      "grad_norm": 1.5231815576553345,
+      "kl": 1.4100277423858643,
+      "learning_rate": 3.960624164093587e-06,
+      "loss": 0.0564,
+      "reward": 3.6343533992767334,
+      "reward_std": 0.24779343605041504,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.6343532800674438,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 594
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 23.8,
+      "grad_norm": 0.5205547213554382,
+      "kl": 1.2938545942306519,
+      "learning_rate": 3.956194242169506e-06,
+      "loss": 0.0518,
+      "reward": 3.1121644973754883,
+      "reward_std": 0.49655723571777344,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.36216452717781067,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 595
+    },
+    {
+      "completion_length": 606.0,
+      "epoch": 23.84,
+      "grad_norm": 1.4387381076812744,
+      "kl": 0.6888614892959595,
+      "learning_rate": 3.951757389277349e-06,
+      "loss": 0.0276,
+      "reward": 3.201009750366211,
+      "reward_std": 0.42461612820625305,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.2010095715522766,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 596
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 23.88,
+      "grad_norm": 0.6438161730766296,
+      "kl": 1.303391695022583,
+      "learning_rate": 3.947313626534965e-06,
+      "loss": 0.0521,
+      "reward": 2.792616844177246,
+      "reward_std": 0.48476383090019226,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.042616650462150574,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 597
+    },
+    {
+      "completion_length": 696.0,
+      "epoch": 23.92,
+      "grad_norm": 0.4826262891292572,
+      "kl": 0.8998121023178101,
+      "learning_rate": 3.942862975093085e-06,
+      "loss": 0.036,
+      "reward": 3.009612798690796,
+      "reward_std": 0.4383618235588074,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.1346127986907959,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 598
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 23.96,
+      "grad_norm": 0.3816401958465576,
+      "kl": 0.34214648604393005,
+      "learning_rate": 3.938405456135231e-06,
+      "loss": 0.0137,
+      "reward": -1.53125,
+      "reward_std": 0.5436661839485168,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.96875,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 599
+    },
+    {
+      "completion_length": 624.0,
+      "epoch": 24.0,
+      "grad_norm": 0.48791933059692383,
+      "kl": 0.9707455039024353,
+      "learning_rate": 3.933941090877615e-06,
+      "loss": 0.0388,
+      "reward": 2.9003190994262695,
+      "reward_std": 0.7066404819488525,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9642857313156128,
+      "rewards/wrapped_driving_reward": 0.4360334277153015,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 600
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 24.04,
+      "grad_norm": 0.4375697672367096,
+      "kl": 0.7251780033111572,
+      "learning_rate": 3.929469900569031e-06,
+      "loss": 0.029,
+      "reward": 1.5899887084960938,
+      "reward_std": 0.8056603670120239,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9583333134651184,
+      "rewards/wrapped_driving_reward": -1.1183446645736694,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 601
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 24.08,
+      "grad_norm": 0.6219246983528137,
+      "kl": 0.916982114315033,
+      "learning_rate": 3.924991906490758e-06,
+      "loss": 0.0367,
+      "reward": 1.9977405071258545,
+      "reward_std": 1.210959553718567,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.37725937366485596,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 602
+    },
+    {
+      "completion_length": 617.0,
+      "epoch": 24.12,
+      "grad_norm": 0.5328007340431213,
+      "kl": 0.8875964283943176,
+      "learning_rate": 3.92050712995646e-06,
+      "loss": 0.0355,
+      "reward": 3.042351245880127,
+      "reward_std": 0.12825651466846466,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.04235142096877098,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 603
+    },
+    {
+      "completion_length": 616.0,
+      "epoch": 24.16,
+      "grad_norm": 0.5093056559562683,
+      "kl": 1.141879677772522,
+      "learning_rate": 3.916015592312083e-06,
+      "loss": 0.0457,
+      "reward": 2.9928762912750244,
+      "reward_std": 0.5530683994293213,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.11787624657154083,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 604
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 24.2,
+      "grad_norm": 0.4226742386817932,
+      "kl": 1.135231614112854,
+      "learning_rate": 3.911517314935752e-06,
+      "loss": 0.0454,
+      "reward": 2.794490098953247,
+      "reward_std": 0.5061540007591248,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.04449019581079483,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 605
+    },
+    {
+      "completion_length": 617.0,
+      "epoch": 24.24,
+      "grad_norm": 0.5197967290878296,
+      "kl": 1.5424551963806152,
+      "learning_rate": 3.907012319237672e-06,
+      "loss": 0.0617,
+      "reward": 0.6565590500831604,
+      "reward_std": 1.6452194452285767,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.949999988079071,
+      "rewards/wrapped_driving_reward": -1.7934409379959106,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 606
+    },
+    {
+      "completion_length": 651.0,
+      "epoch": 24.28,
+      "grad_norm": 0.551070511341095,
+      "kl": 1.370877981185913,
+      "learning_rate": 3.902500626660025e-06,
+      "loss": 0.0548,
+      "reward": 3.000037670135498,
+      "reward_std": 0.07534631341695786,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9772727489471436,
+      "rewards/wrapped_driving_reward": 0.022764792665839195,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 607
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 24.32,
+      "grad_norm": 1.6095973253250122,
+      "kl": 2.3942067623138428,
+      "learning_rate": 3.897982258676867e-06,
+      "loss": 0.0958,
+      "reward": 1.3801480531692505,
+      "reward_std": 3.5869948863983154,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.8698518872261047,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 608
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 24.36,
+      "grad_norm": 0.35611942410469055,
+      "kl": 1.5788023471832275,
+      "learning_rate": 3.8934572367940285e-06,
+      "loss": 0.0632,
+      "reward": 3.4379539489746094,
+      "reward_std": 0.21014034748077393,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9791666865348816,
+      "rewards/wrapped_driving_reward": 0.5837870836257935,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 609
+    },
+    {
+      "completion_length": 452.0,
+      "epoch": 24.4,
+      "grad_norm": 0.5418919920921326,
+      "kl": 0.8879813551902771,
+      "learning_rate": 3.888925582549006e-06,
+      "loss": 0.0355,
+      "reward": 2.4353699684143066,
+      "reward_std": 0.7164378762245178,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.5646300315856934,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 610
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 24.44,
+      "grad_norm": 0.6341983079910278,
+      "kl": 1.4851973056793213,
+      "learning_rate": 3.8843873175108685e-06,
+      "loss": 0.0594,
+      "reward": 3.246654987335205,
+      "reward_std": 0.551415205001831,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.930555522441864,
+      "rewards/wrapped_driving_reward": 0.6910994052886963,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 611
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 24.48,
+      "grad_norm": 0.6838855147361755,
+      "kl": 1.131606936454773,
+      "learning_rate": 3.879842463280146e-06,
+      "loss": 0.0453,
+      "reward": -1.75,
+      "reward_std": 0.5,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 612
+    },
+    {
+      "completion_length": 484.0,
+      "epoch": 24.52,
+      "grad_norm": 0.5183650851249695,
+      "kl": 1.2218328714370728,
+      "learning_rate": 3.875291041488734e-06,
+      "loss": 0.0489,
+      "reward": 3.104029655456543,
+      "reward_std": 0.5371575951576233,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.229029580950737,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 613
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 24.56,
+      "grad_norm": 0.39312297105789185,
+      "kl": 1.4679391384124756,
+      "learning_rate": 3.870733073799785e-06,
+      "loss": 0.0587,
+      "reward": 1.5885634422302246,
+      "reward_std": 3.0621588230133057,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9114365577697754,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 614
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 24.6,
+      "grad_norm": 0.4172166883945465,
+      "kl": 1.2957038879394531,
+      "learning_rate": 3.866168581907609e-06,
+      "loss": 0.0518,
+      "reward": 2.2104885578155518,
+      "reward_std": 0.834479033946991,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.414511501789093,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 615
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 24.64,
+      "grad_norm": 1.0627772808074951,
+      "kl": 0.836889922618866,
+      "learning_rate": 3.861597587537568e-06,
+      "loss": 0.0335,
+      "reward": -2.125,
+      "reward_std": 1.314977765083313,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 616
+    },
+    {
+      "completion_length": 686.0,
+      "epoch": 24.68,
+      "grad_norm": 0.41289910674095154,
+      "kl": 1.7737808227539062,
+      "learning_rate": 3.8570201124459745e-06,
+      "loss": 0.071,
+      "reward": 2.3616647720336914,
+      "reward_std": 0.9351847171783447,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.6383353471755981,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 617
+    },
+    {
+      "completion_length": 581.0,
+      "epoch": 24.72,
+      "grad_norm": 0.779534101486206,
+      "kl": 1.1413559913635254,
+      "learning_rate": 3.8524361784199855e-06,
+      "loss": 0.0457,
+      "reward": 3.272754192352295,
+      "reward_std": 0.6635096669197083,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9375,
+      "rewards/wrapped_driving_reward": 0.5852542519569397,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 618
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 24.76,
+      "grad_norm": 0.42929840087890625,
+      "kl": 1.5810688734054565,
+      "learning_rate": 3.847845807277501e-06,
+      "loss": 0.0632,
+      "reward": -1.5,
+      "reward_std": 0.5773502588272095,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 619
+    },
+    {
+      "completion_length": 557.0,
+      "epoch": 24.8,
+      "grad_norm": 0.42557451128959656,
+      "kl": 0.7237415313720703,
+      "learning_rate": 3.8432490208670605e-06,
+      "loss": 0.0289,
+      "reward": 3.082545280456543,
+      "reward_std": 0.1998668611049652,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.0825452134013176,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 620
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 24.84,
+      "grad_norm": 0.5002950429916382,
+      "kl": 1.1155179738998413,
+      "learning_rate": 3.838645841067735e-06,
+      "loss": 0.0446,
+      "reward": -1.149999976158142,
+      "reward_std": 0.29999998211860657,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8500000238418579,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 621
+    },
+    {
+      "completion_length": 615.0,
+      "epoch": 24.88,
+      "grad_norm": 0.44429194927215576,
+      "kl": 1.1357996463775635,
+      "learning_rate": 3.83403628978903e-06,
+      "loss": 0.0454,
+      "reward": 3.2363839149475098,
+      "reward_std": 0.5219359993934631,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.36138415336608887,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 622
+    },
+    {
+      "completion_length": 665.0,
+      "epoch": 24.92,
+      "grad_norm": 0.41479218006134033,
+      "kl": 1.0186771154403687,
+      "learning_rate": 3.829420388970772e-06,
+      "loss": 0.0407,
+      "reward": 2.8114256858825684,
+      "reward_std": 0.4211501479148865,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.06142570078372955,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 623
+    },
+    {
+      "completion_length": 620.0,
+      "epoch": 24.96,
+      "grad_norm": 0.5567065477371216,
+      "kl": 0.8860252499580383,
+      "learning_rate": 3.824798160583012e-06,
+      "loss": 0.0354,
+      "reward": -0.21129226684570312,
+      "reward_std": 2.2909140586853027,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.987500011920929,
+      "rewards/wrapped_driving_reward": -2.9487922191619873,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 624
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 25.0,
+      "grad_norm": 0.6249330043792725,
+      "kl": 1.2577399015426636,
+      "learning_rate": 3.82016962662592e-06,
+      "loss": 0.0503,
+      "reward": -1.1666667461395264,
+      "reward_std": 0.235702246427536,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9583333134651184,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 625
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 25.04,
+      "grad_norm": 0.4327258765697479,
+      "kl": 1.0329591035842896,
+      "learning_rate": 3.815534809129674e-06,
+      "loss": 0.0413,
+      "reward": -0.3447999358177185,
+      "reward_std": 3.6673803329467773,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -2.0947999954223633,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 626
+    },
+    {
+      "completion_length": 735.0,
+      "epoch": 25.08,
+      "grad_norm": 0.9782822728157043,
+      "kl": 1.77318274974823,
+      "learning_rate": 3.8108937301543613e-06,
+      "loss": 0.0709,
+      "reward": -1.25,
+      "reward_std": 0.28867512941360474,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 627
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 25.12,
+      "grad_norm": 0.4559270143508911,
+      "kl": 0.4221249222755432,
+      "learning_rate": 3.806246411789872e-06,
+      "loss": 0.0169,
+      "reward": 1.5232298374176025,
+      "reward_std": 3.688981056213379,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.6017701625823975,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 628
+    },
+    {
+      "completion_length": 523.0,
+      "epoch": 25.16,
+      "grad_norm": 2.418384552001953,
+      "kl": 1.021903395652771,
+      "learning_rate": 3.8015928761557937e-06,
+      "loss": 0.0409,
+      "reward": -1.5,
+      "reward_std": 0.40824830532073975,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 629
+    },
+    {
+      "completion_length": 481.0,
+      "epoch": 25.2,
+      "grad_norm": 1.3654940128326416,
+      "kl": 0.7937284708023071,
+      "learning_rate": 3.796933145401304e-06,
+      "loss": 0.0317,
+      "reward": 2.0573642253875732,
+      "reward_std": 3.373218059539795,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.44263583421707153,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 630
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 25.24,
+      "grad_norm": 6.166470527648926,
+      "kl": 1.57492995262146,
+      "learning_rate": 3.7922672417050687e-06,
+      "loss": 0.063,
+      "reward": 3.1056036949157715,
+      "reward_std": 0.28644248843193054,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9750000238418579,
+      "rewards/wrapped_driving_reward": 0.2556036412715912,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 631
+    },
+    {
+      "completion_length": 444.0,
+      "epoch": 25.28,
+      "grad_norm": 0.11741337180137634,
+      "kl": 0.7082597017288208,
+      "learning_rate": 3.787595187275136e-06,
+      "loss": 0.0283,
+      "reward": -1.0,
+      "reward_std": 0.0,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 632
+    },
+    {
+      "completion_length": 671.0,
+      "epoch": 25.32,
+      "grad_norm": 0.5336841344833374,
+      "kl": 1.2229433059692383,
+      "learning_rate": 3.782917004348826e-06,
+      "loss": 0.0489,
+      "reward": 2.960865020751953,
+      "reward_std": 0.3422490060329437,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.08586501330137253,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 633
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 25.36,
+      "grad_norm": 0.7414114475250244,
+      "kl": 1.5562310218811035,
+      "learning_rate": 3.77823271519263e-06,
+      "loss": 0.0622,
+      "reward": 1.009045124053955,
+      "reward_std": 3.342817783355713,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.625,
+      "rewards/wrapped_driving_reward": -0.7409549355506897,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 634
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 25.4,
+      "grad_norm": 0.40905439853668213,
+      "kl": 1.2026870250701904,
+      "learning_rate": 3.773542342102105e-06,
+      "loss": 0.0481,
+      "reward": 3.4216301441192627,
+      "reward_std": 0.28488659858703613,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.4216301441192627,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 635
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 25.44,
+      "grad_norm": 0.44360411167144775,
+      "kl": 1.3418991565704346,
+      "learning_rate": 3.768845907401761e-06,
+      "loss": 0.0537,
+      "reward": 1.978645920753479,
+      "reward_std": 2.0448219776153564,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.8963539600372314,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 636
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 25.48,
+      "grad_norm": 0.5750876069068909,
+      "kl": 1.2612106800079346,
+      "learning_rate": 3.764143433444962e-06,
+      "loss": 0.0504,
+      "reward": -1.75,
+      "reward_std": 1.1902379989624023,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 637
+    },
+    {
+      "completion_length": 593.0,
+      "epoch": 25.52,
+      "grad_norm": 0.6619950532913208,
+      "kl": 1.4557205438613892,
+      "learning_rate": 3.759434942613816e-06,
+      "loss": 0.0582,
+      "reward": 2.910365104675293,
+      "reward_std": 0.15222014486789703,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.03536504879593849,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 638
+    },
+    {
+      "completion_length": 445.0,
+      "epoch": 25.56,
+      "grad_norm": 0.5441882610321045,
+      "kl": 0.7263493537902832,
+      "learning_rate": 3.75472045731907e-06,
+      "loss": 0.0291,
+      "reward": 1.806579828262329,
+      "reward_std": 1.926430344581604,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.068420171737671,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 639
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 25.6,
+      "grad_norm": 0.37299302220344543,
+      "kl": 1.4050265550613403,
+      "learning_rate": 3.7500000000000005e-06,
+      "loss": 0.0562,
+      "reward": 3.081347703933716,
+      "reward_std": 0.26734286546707153,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.08134761452674866,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 640
+    },
+    {
+      "completion_length": 658.0,
+      "epoch": 25.64,
+      "grad_norm": 0.4773707687854767,
+      "kl": 1.1113680601119995,
+      "learning_rate": 3.7452735931243108e-06,
+      "loss": 0.0445,
+      "reward": 3.2923827171325684,
+      "reward_std": 0.3292248547077179,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.2923825681209564,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 641
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 25.68,
+      "grad_norm": 0.47026556730270386,
+      "kl": 1.7575290203094482,
+      "learning_rate": 3.7405412591880213e-06,
+      "loss": 0.0703,
+      "reward": -1.774999976158142,
+      "reward_std": 1.4840823411941528,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7250000238418579,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 642
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 25.72,
+      "grad_norm": 0.4005660116672516,
+      "kl": 1.021146297454834,
+      "learning_rate": 3.735803020715362e-06,
+      "loss": 0.0408,
+      "reward": 0.7224711775779724,
+      "reward_std": 1.9892507791519165,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -2.152529001235962,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 643
+    },
+    {
+      "completion_length": 492.0,
+      "epoch": 25.76,
+      "grad_norm": 0.9656462669372559,
+      "kl": 0.7833800315856934,
+      "learning_rate": 3.7310589002586683e-06,
+      "loss": 0.0313,
+      "reward": 1.5308680534362793,
+      "reward_std": 3.0411288738250732,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9691319465637207,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 644
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 25.8,
+      "grad_norm": 0.36192595958709717,
+      "kl": 1.2042280435562134,
+      "learning_rate": 3.7263089203982698e-06,
+      "loss": 0.0482,
+      "reward": 0.9078962802886963,
+      "reward_std": 2.2103357315063477,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.8421037197113037,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 645
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 25.84,
+      "grad_norm": 0.38677889108657837,
+      "kl": 1.4992326498031616,
+      "learning_rate": 3.721553103742388e-06,
+      "loss": 0.06,
+      "reward": 0.9938499927520752,
+      "reward_std": 3.3306725025177,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.1311500072479248,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 646
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 25.88,
+      "grad_norm": 0.6154844164848328,
+      "kl": 1.3798651695251465,
+      "learning_rate": 3.7167914729270205e-06,
+      "loss": 0.0552,
+      "reward": 2.4758121967315674,
+      "reward_std": 0.3415810167789459,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9791666865348816,
+      "rewards/wrapped_driving_reward": -0.2533544898033142,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 647
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 25.92,
+      "grad_norm": 0.3917384743690491,
+      "kl": 0.8261408805847168,
+      "learning_rate": 3.7120240506158433e-06,
+      "loss": 0.033,
+      "reward": -0.25806379318237305,
+      "reward_std": 1.679756999015808,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9285714626312256,
+      "rewards/wrapped_driving_reward": -3.1866352558135986,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 648
+    },
+    {
+      "completion_length": 562.0,
+      "epoch": 25.96,
+      "grad_norm": 0.5384634137153625,
+      "kl": 1.1905111074447632,
+      "learning_rate": 3.7072508595000935e-06,
+      "loss": 0.0476,
+      "reward": 2.2796902656555176,
+      "reward_std": 0.8608195781707764,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8678571581840515,
+      "rewards/wrapped_driving_reward": -0.46316689252853394,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 649
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 26.0,
+      "grad_norm": 0.36869606375694275,
+      "kl": 1.1743124723434448,
+      "learning_rate": 3.7024719222984696e-06,
+      "loss": 0.047,
+      "reward": 2.691631317138672,
+      "reward_std": 0.5718726515769958,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.84375,
+      "rewards/wrapped_driving_reward": 0.09788113832473755,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 650
+    },
+    {
+      "completion_length": 582.0,
+      "epoch": 26.04,
+      "grad_norm": 0.8777485489845276,
+      "kl": 1.086169719696045,
+      "learning_rate": 3.6976872617570163e-06,
+      "loss": 0.0434,
+      "reward": 3.3562309741973877,
+      "reward_std": 0.4827421307563782,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.6062309741973877,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 651
+    },
+    {
+      "completion_length": 695.0,
+      "epoch": 26.08,
+      "grad_norm": 0.8116427063941956,
+      "kl": 1.1721394062042236,
+      "learning_rate": 3.6928969006490212e-06,
+      "loss": 0.0469,
+      "reward": 3.1609246730804443,
+      "reward_std": 0.4948073923587799,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.2859245240688324,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 652
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 26.12,
+      "grad_norm": 0.6845850348472595,
+      "kl": 0.6479750275611877,
+      "learning_rate": 3.6881008617749042e-06,
+      "loss": 0.0259,
+      "reward": 0.7977969646453857,
+      "reward_std": 3.241457223892212,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7083333134651184,
+      "rewards/wrapped_driving_reward": -0.9105363488197327,
+      "rewards/wrapped_format_reward": 0.25,
+      "step": 653
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 26.16,
+      "grad_norm": 0.402665376663208,
+      "kl": 1.176146388053894,
+      "learning_rate": 3.6832991679621087e-06,
+      "loss": 0.047,
+      "reward": -1.3214285373687744,
+      "reward_std": 0.38905078172683716,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9285714626312256,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 654
+    },
+    {
+      "completion_length": 615.0,
+      "epoch": 26.2,
+      "grad_norm": 0.4569839835166931,
+      "kl": 1.0099406242370605,
+      "learning_rate": 3.6784918420649952e-06,
+      "loss": 0.0404,
+      "reward": 2.8477678298950195,
+      "reward_std": 0.21331287920475006,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.02723217010498047,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 655
+    },
+    {
+      "completion_length": 448.0,
+      "epoch": 26.24,
+      "grad_norm": 0.6247033476829529,
+      "kl": 0.5885288715362549,
+      "learning_rate": 3.6736789069647273e-06,
+      "loss": 0.0235,
+      "reward": 2.829052448272705,
+      "reward_std": 0.3819313943386078,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8157051205635071,
+      "rewards/wrapped_driving_reward": 0.2633473873138428,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 656
+    },
+    {
+      "completion_length": 526.0,
+      "epoch": 26.28,
+      "grad_norm": 0.5222458839416504,
+      "kl": 1.7896028757095337,
+      "learning_rate": 3.6688603855691713e-06,
+      "loss": 0.0716,
+      "reward": 2.5255279541015625,
+      "reward_std": 0.7786571979522705,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.7613636255264282,
+      "rewards/wrapped_driving_reward": 0.014164302498102188,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 657
+    },
+    {
+      "completion_length": 745.0,
+      "epoch": 26.32,
+      "grad_norm": 0.6088178753852844,
+      "kl": 1.4155036211013794,
+      "learning_rate": 3.664036300812779e-06,
+      "loss": 0.0566,
+      "reward": 1.3521000146865845,
+      "reward_std": 3.5682103633880615,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.8978999853134155,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 658
+    },
+    {
+      "completion_length": 686.0,
+      "epoch": 26.36,
+      "grad_norm": 1.1373714208602905,
+      "kl": 0.8373534083366394,
+      "learning_rate": 3.6592066756564825e-06,
+      "loss": 0.0335,
+      "reward": -1.25,
+      "reward_std": 0.5,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 659
+    },
+    {
+      "completion_length": 613.0,
+      "epoch": 26.4,
+      "grad_norm": 0.40486952662467957,
+      "kl": 0.6469566226005554,
+      "learning_rate": 3.654371533087586e-06,
+      "loss": 0.0259,
+      "reward": -0.7124611735343933,
+      "reward_std": 2.128596067428589,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -3.212461233139038,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 660
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 26.44,
+      "grad_norm": 0.4986326992511749,
+      "kl": 1.450907588005066,
+      "learning_rate": 3.64953089611965e-06,
+      "loss": 0.058,
+      "reward": 2.939197301864624,
+      "reward_std": 0.35693034529685974,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.884615421295166,
+      "rewards/wrapped_driving_reward": 0.17958186566829681,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 661
+    },
+    {
+      "completion_length": 533.0,
+      "epoch": 26.48,
+      "grad_norm": 0.6424423456192017,
+      "kl": 1.0434520244598389,
+      "learning_rate": 3.6446847877923917e-06,
+      "loss": 0.0417,
+      "reward": 2.148531913757324,
+      "reward_std": 0.4976847171783447,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.8514681458473206,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 662
+    },
+    {
+      "completion_length": 495.0,
+      "epoch": 26.52,
+      "grad_norm": 0.4873075783252716,
+      "kl": 0.7196347713470459,
+      "learning_rate": 3.639833231171569e-06,
+      "loss": 0.0288,
+      "reward": 2.8510382175445557,
+      "reward_std": 0.5779574513435364,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.023961812257766724,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 663
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 26.56,
+      "grad_norm": 0.46740320324897766,
+      "kl": 1.3179888725280762,
+      "learning_rate": 3.634976249348867e-06,
+      "loss": 0.0527,
+      "reward": 0.5123102068901062,
+      "reward_std": 3.0649352073669434,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.7083333134651184,
+      "rewards/wrapped_driving_reward": -1.6960229873657227,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 664
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 26.6,
+      "grad_norm": 0.40568238496780396,
+      "kl": 1.0916098356246948,
+      "learning_rate": 3.6301138654418e-06,
+      "loss": 0.0437,
+      "reward": 2.6497511863708496,
+      "reward_std": 0.5616340041160583,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.27475130558013916,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 665
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 26.64,
+      "grad_norm": 0.43641456961631775,
+      "kl": 1.1734923124313354,
+      "learning_rate": 3.625246102593588e-06,
+      "loss": 0.0469,
+      "reward": 0.9107986688613892,
+      "reward_std": 3.306107759475708,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.0892013311386108,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 666
+    },
+    {
+      "completion_length": 531.0,
+      "epoch": 26.68,
+      "grad_norm": 0.0914665013551712,
+      "kl": 0.9140002727508545,
+      "learning_rate": 3.6203729839730567e-06,
+      "loss": 0.0366,
+      "reward": -1.0,
+      "reward_std": 0.0,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 667
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 26.72,
+      "grad_norm": 0.5565376877784729,
+      "kl": 2.005063772201538,
+      "learning_rate": 3.6154945327745223e-06,
+      "loss": 0.0802,
+      "reward": -1.375,
+      "reward_std": 0.4787135720252991,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 668
+    },
+    {
+      "completion_length": 716.0,
+      "epoch": 26.76,
+      "grad_norm": 0.5163022875785828,
+      "kl": 1.0322647094726562,
+      "learning_rate": 3.610610772217682e-06,
+      "loss": 0.0413,
+      "reward": 1.4180800914764404,
+      "reward_std": 1.708691954612732,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.5819199085235596,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 669
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 26.8,
+      "grad_norm": 0.5534092783927917,
+      "kl": 0.7664154171943665,
+      "learning_rate": 3.6057217255475034e-06,
+      "loss": 0.0307,
+      "reward": 3.130662441253662,
+      "reward_std": 0.6534955501556396,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.925000011920929,
+      "rewards/wrapped_driving_reward": 0.5806624889373779,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 670
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 26.84,
+      "grad_norm": 0.5211420655250549,
+      "kl": 1.697385311126709,
+      "learning_rate": 3.600827416034115e-06,
+      "loss": 0.0679,
+      "reward": 1.2811267375946045,
+      "reward_std": 3.542926549911499,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.737500011920929,
+      "rewards/wrapped_driving_reward": -0.8313732743263245,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 671
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 26.88,
+      "grad_norm": 0.4372680187225342,
+      "kl": 1.1519041061401367,
+      "learning_rate": 3.595927866972694e-06,
+      "loss": 0.0461,
+      "reward": 2.822664260864258,
+      "reward_std": 0.1902877241373062,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.19766449928283691,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 672
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 26.92,
+      "grad_norm": 0.4989381730556488,
+      "kl": 1.5017751455307007,
+      "learning_rate": 3.591023101683355e-06,
+      "loss": 0.0601,
+      "reward": -1.0277777910232544,
+      "reward_std": 0.05555558204650879,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9722222089767456,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 673
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 26.96,
+      "grad_norm": 0.58172607421875,
+      "kl": 1.2829159498214722,
+      "learning_rate": 3.586113143511043e-06,
+      "loss": 0.0513,
+      "reward": 2.83713960647583,
+      "reward_std": 0.4975849390029907,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.03786037862300873,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 674
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 27.0,
+      "grad_norm": 0.49302029609680176,
+      "kl": 1.1698321104049683,
+      "learning_rate": 3.5811980158254156e-06,
+      "loss": 0.0468,
+      "reward": 1.1822489500045776,
+      "reward_std": 3.4572482109069824,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.0677510499954224,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 675
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 27.04,
+      "grad_norm": 0.4445314407348633,
+      "kl": 1.0804122686386108,
+      "learning_rate": 3.5762777420207382e-06,
+      "loss": 0.0432,
+      "reward": -1.1875,
+      "reward_std": 0.23935678601264954,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9375,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 676
+    },
+    {
+      "completion_length": 489.0,
+      "epoch": 27.08,
+      "grad_norm": 0.4960654675960541,
+      "kl": 1.2753853797912598,
+      "learning_rate": 3.5713523455157686e-06,
+      "loss": 0.051,
+      "reward": 0.35062074661254883,
+      "reward_std": 1.9192057847976685,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -2.524379253387451,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 677
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 27.12,
+      "grad_norm": 0.3840659260749817,
+      "kl": 0.7700368165969849,
+      "learning_rate": 3.566421849753646e-06,
+      "loss": 0.0308,
+      "reward": 1.9260783195495605,
+      "reward_std": 1.246964931488037,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.5739217400550842,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 678
+    },
+    {
+      "completion_length": 637.0,
+      "epoch": 27.16,
+      "grad_norm": 0.3894844651222229,
+      "kl": 0.9075636863708496,
+      "learning_rate": 3.5614862782017833e-06,
+      "loss": 0.0363,
+      "reward": 2.386613368988037,
+      "reward_std": 1.434007167816162,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9772727489471436,
+      "rewards/wrapped_driving_reward": -0.5906594395637512,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 679
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 27.2,
+      "grad_norm": 2.2594847679138184,
+      "kl": 1.5272691249847412,
+      "learning_rate": 3.556545654351749e-06,
+      "loss": 0.0611,
+      "reward": 3.106175422668457,
+      "reward_std": 0.4387306869029999,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9821428656578064,
+      "rewards/wrapped_driving_reward": 0.12403266131877899,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 680
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 27.24,
+      "grad_norm": 0.37824928760528564,
+      "kl": 1.3627076148986816,
+      "learning_rate": 3.551600001719161e-06,
+      "loss": 0.0545,
+      "reward": -1.2204545736312866,
+      "reward_std": 0.20708855986595154,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9045454263687134,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 681
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 27.28,
+      "grad_norm": 0.3919558525085449,
+      "kl": 1.2976757287979126,
+      "learning_rate": 3.5466493438435707e-06,
+      "loss": 0.0519,
+      "reward": -1.375,
+      "reward_std": 0.4787135720252991,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.875,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 682
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 27.32,
+      "grad_norm": 0.49715036153793335,
+      "kl": 0.9081407189369202,
+      "learning_rate": 3.541693704288355e-06,
+      "loss": 0.0363,
+      "reward": -1.25,
+      "reward_std": 0.5,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 683
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 27.36,
+      "grad_norm": 0.540151834487915,
+      "kl": 1.1362556219100952,
+      "learning_rate": 3.536733106640598e-06,
+      "loss": 0.0455,
+      "reward": 1.1239418983459473,
+      "reward_std": 3.4454872608184814,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.0010579824447632,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 684
+    },
+    {
+      "completion_length": 564.0,
+      "epoch": 27.4,
+      "grad_norm": 0.41077589988708496,
+      "kl": 1.017673134803772,
+      "learning_rate": 3.531767574510987e-06,
+      "loss": 0.0407,
+      "reward": 2.436887264251709,
+      "reward_std": 0.47174349427223206,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.43811261653900146,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 685
+    },
+    {
+      "completion_length": 687.0,
+      "epoch": 27.44,
+      "grad_norm": 0.3432943522930145,
+      "kl": 1.0499712228775024,
+      "learning_rate": 3.5267971315336936e-06,
+      "loss": 0.042,
+      "reward": 2.8152427673339844,
+      "reward_std": 0.34102752804756165,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9722222089767456,
+      "rewards/wrapped_driving_reward": 0.21802052855491638,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 686
+    },
+    {
+      "completion_length": 644.0,
+      "epoch": 27.48,
+      "grad_norm": 0.41452446579933167,
+      "kl": 0.9771077632904053,
+      "learning_rate": 3.5218218013662626e-06,
+      "loss": 0.0391,
+      "reward": 2.786472797393799,
+      "reward_std": 0.441410094499588,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.036472804844379425,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 687
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 27.52,
+      "grad_norm": 0.3727835714817047,
+      "kl": 1.0077673196792603,
+      "learning_rate": 3.516841607689501e-06,
+      "loss": 0.0403,
+      "reward": 2.619081974029541,
+      "reward_std": 0.729070246219635,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.25591808557510376,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 688
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 27.56,
+      "grad_norm": 0.38604769110679626,
+      "kl": 0.7550218105316162,
+      "learning_rate": 3.511856574207364e-06,
+      "loss": 0.0302,
+      "reward": -0.4012797772884369,
+      "reward_std": 3.317986488342285,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -2.276279926300049,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 689
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 27.6,
+      "grad_norm": 0.7709413170814514,
+      "kl": 1.3966618776321411,
+      "learning_rate": 3.5068667246468437e-06,
+      "loss": 0.0559,
+      "reward": 3.257936716079712,
+      "reward_std": 0.17135444283485413,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.2579367756843567,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 690
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 27.64,
+      "grad_norm": 0.4005296230316162,
+      "kl": 0.7381643056869507,
+      "learning_rate": 3.5018720827578523e-06,
+      "loss": 0.0295,
+      "reward": 1.0149767398834229,
+      "reward_std": 1.716369867324829,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8794642686843872,
+      "rewards/wrapped_driving_reward": -1.3644875288009644,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 691
+    },
+    {
+      "completion_length": 538.0,
+      "epoch": 27.68,
+      "grad_norm": 0.6042305827140808,
+      "kl": 0.7873980402946472,
+      "learning_rate": 3.496872672313116e-06,
+      "loss": 0.0315,
+      "reward": 3.5595736503601074,
+      "reward_std": 0.09154798090457916,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9303977489471436,
+      "rewards/wrapped_driving_reward": 0.6291758418083191,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 692
+    },
+    {
+      "completion_length": 660.0,
+      "epoch": 27.72,
+      "grad_norm": 1.7775932550430298,
+      "kl": 0.5605735778808594,
+      "learning_rate": 3.491868517108053e-06,
+      "loss": 0.0224,
+      "reward": 3.6727917194366455,
+      "reward_std": 0.36999544501304626,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9642857313156128,
+      "rewards/wrapped_driving_reward": 0.7085059285163879,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 693
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 27.76,
+      "grad_norm": 0.5017414093017578,
+      "kl": 0.5329836010932922,
+      "learning_rate": 3.486859640960668e-06,
+      "loss": 0.0213,
+      "reward": 2.7044076919555664,
+      "reward_std": 0.31959784030914307,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.17059244215488434,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 694
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 27.8,
+      "grad_norm": 0.5945215225219727,
+      "kl": 2.0084989070892334,
+      "learning_rate": 3.481846067711436e-06,
+      "loss": 0.0803,
+      "reward": 2.8925669193267822,
+      "reward_std": 0.6678995490074158,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.39256682991981506,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 695
+    },
+    {
+      "completion_length": 566.0,
+      "epoch": 27.84,
+      "grad_norm": 0.4806859791278839,
+      "kl": 1.042332649230957,
+      "learning_rate": 3.476827821223184e-06,
+      "loss": 0.0417,
+      "reward": 3.4408185482025146,
+      "reward_std": 0.12357556074857712,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.44081851840019226,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 696
+    },
+    {
+      "completion_length": 737.0,
+      "epoch": 27.88,
+      "grad_norm": 0.638775646686554,
+      "kl": 1.0922187566757202,
+      "learning_rate": 3.4718049253809894e-06,
+      "loss": 0.0437,
+      "reward": 2.549114465713501,
+      "reward_std": 0.2564372420310974,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.07588561624288559,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 697
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 27.92,
+      "grad_norm": 0.4154502749443054,
+      "kl": 0.6086506843566895,
+      "learning_rate": 3.466777404092052e-06,
+      "loss": 0.0243,
+      "reward": -1.25,
+      "reward_std": 0.5,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 698
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 27.96,
+      "grad_norm": 0.41160765290260315,
+      "kl": 1.0010474920272827,
+      "learning_rate": 3.4617452812855908e-06,
+      "loss": 0.04,
+      "reward": 1.1086905002593994,
+      "reward_std": 3.1339128017425537,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.0163094997406006,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 699
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 28.0,
+      "grad_norm": 0.5900842547416687,
+      "kl": 0.7559553384780884,
+      "learning_rate": 3.4567085809127247e-06,
+      "loss": 0.0302,
+      "reward": 2.649343729019165,
+      "reward_std": 0.5213807225227356,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.3506563901901245,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 700
+    },
+    {
+      "completion_length": 696.0,
+      "epoch": 28.04,
+      "grad_norm": 0.360720157623291,
+      "kl": 0.8732097744941711,
+      "learning_rate": 3.4516673269463617e-06,
+      "loss": 0.0349,
+      "reward": 2.775423049926758,
+      "reward_std": 0.3682582676410675,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9583333134651184,
+      "rewards/wrapped_driving_reward": 0.06708974391222,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 701
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 28.08,
+      "grad_norm": 0.47541409730911255,
+      "kl": 1.2193118333816528,
+      "learning_rate": 3.4466215433810827e-06,
+      "loss": 0.0488,
+      "reward": -0.1097484827041626,
+      "reward_std": 2.7219316959381104,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -2.359748363494873,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 702
+    },
+    {
+      "completion_length": 426.0,
+      "epoch": 28.12,
+      "grad_norm": 0.49380800127983093,
+      "kl": 0.5956392884254456,
+      "learning_rate": 3.441571254233027e-06,
+      "loss": 0.0238,
+      "reward": 2.9899373054504395,
+      "reward_std": 0.14259079098701477,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.010062739253044128,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 703
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 28.16,
+      "grad_norm": 0.4521055817604065,
+      "kl": 1.294080376625061,
+      "learning_rate": 3.436516483539781e-06,
+      "loss": 0.0518,
+      "reward": -1.2708332538604736,
+      "reward_std": 0.4876958429813385,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9791666865348816,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 704
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 28.2,
+      "grad_norm": 0.4913451373577118,
+      "kl": 1.2871887683868408,
+      "learning_rate": 3.4314572553602577e-06,
+      "loss": 0.0515,
+      "reward": -1.524999976158142,
+      "reward_std": 0.04999999329447746,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9750000238418579,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 705
+    },
+    {
+      "completion_length": 492.0,
+      "epoch": 28.24,
+      "grad_norm": 0.4417870044708252,
+      "kl": 0.6419669985771179,
+      "learning_rate": 3.426393593774591e-06,
+      "loss": 0.0257,
+      "reward": -1.0499999523162842,
+      "reward_std": 0.10000002384185791,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.949999988079071,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 706
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 28.28,
+      "grad_norm": 0.43232548236846924,
+      "kl": 1.069222331047058,
+      "learning_rate": 3.421325522884013e-06,
+      "loss": 0.0428,
+      "reward": 2.791191816329956,
+      "reward_std": 0.3710920810699463,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8999999761581421,
+      "rewards/wrapped_driving_reward": 0.14119189977645874,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 707
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 28.32,
+      "grad_norm": 0.3766399919986725,
+      "kl": 0.7563959360122681,
+      "learning_rate": 3.4162530668107435e-06,
+      "loss": 0.0303,
+      "reward": 3.1373729705810547,
+      "reward_std": 0.3874880373477936,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.3873729705810547,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 708
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 28.36,
+      "grad_norm": 0.4037916660308838,
+      "kl": 1.1408348083496094,
+      "learning_rate": 3.4111762496978753e-06,
+      "loss": 0.0456,
+      "reward": 2.582378625869751,
+      "reward_std": 0.47437480092048645,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.20737852156162262,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 709
+    },
+    {
+      "completion_length": 571.0,
+      "epoch": 28.4,
+      "grad_norm": 0.43852096796035767,
+      "kl": 1.0089608430862427,
+      "learning_rate": 3.406095095709254e-06,
+      "loss": 0.0404,
+      "reward": 1.6179953813552856,
+      "reward_std": 3.1362671852111816,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.6320046186447144,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 710
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 28.44,
+      "grad_norm": 0.5104399919509888,
+      "kl": 1.659734845161438,
+      "learning_rate": 3.401009629029375e-06,
+      "loss": 0.0664,
+      "reward": 3.0378050804138184,
+      "reward_std": 0.20842501521110535,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.03780514374375343,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 711
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 28.48,
+      "grad_norm": 0.4440138041973114,
+      "kl": 1.08497154712677,
+      "learning_rate": 3.39591987386325e-06,
+      "loss": 0.0434,
+      "reward": 1.182783603668213,
+      "reward_std": 3.174339771270752,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.9422163367271423,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 712
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 28.52,
+      "grad_norm": 0.5364874601364136,
+      "kl": 1.3724675178527832,
+      "learning_rate": 3.3908258544363145e-06,
+      "loss": 0.0549,
+      "reward": 1.609416127204895,
+      "reward_std": 3.407630443572998,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.7655838131904602,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 713
+    },
+    {
+      "completion_length": 656.0,
+      "epoch": 28.56,
+      "grad_norm": 0.4007849395275116,
+      "kl": 1.350628137588501,
+      "learning_rate": 3.3857275949942896e-06,
+      "loss": 0.054,
+      "reward": 0.9743061065673828,
+      "reward_std": 1.7018401622772217,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.7756938934326172,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 714
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 28.6,
+      "grad_norm": 0.5446045398712158,
+      "kl": 1.2371965646743774,
+      "learning_rate": 3.3806251198030843e-06,
+      "loss": 0.0495,
+      "reward": 2.513890027999878,
+      "reward_std": 0.7349424958229065,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9166666269302368,
+      "rewards/wrapped_driving_reward": 0.09722331911325455,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 715
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 28.64,
+      "grad_norm": 0.5443968772888184,
+      "kl": 1.1408532857894897,
+      "learning_rate": 3.375518453148669e-06,
+      "loss": 0.0456,
+      "reward": 3.2692654132843018,
+      "reward_std": 0.6697713732719421,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.5192654132843018,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 716
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 28.68,
+      "grad_norm": 0.3911254405975342,
+      "kl": 1.4690412282943726,
+      "learning_rate": 3.370407619336966e-06,
+      "loss": 0.0588,
+      "reward": -1.2867647409439087,
+      "reward_std": 0.33652594685554504,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.8382353186607361,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 717
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 28.72,
+      "grad_norm": 0.403899610042572,
+      "kl": 0.8253454566001892,
+      "learning_rate": 3.3652926426937327e-06,
+      "loss": 0.033,
+      "reward": 1.4513518810272217,
+      "reward_std": 2.3746840953826904,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -1.1736482381820679,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 718
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 28.76,
+      "grad_norm": 0.9827480316162109,
+      "kl": 1.8536406755447388,
+      "learning_rate": 3.360173547564442e-06,
+      "loss": 0.0741,
+      "reward": 3.377265214920044,
+      "reward_std": 0.47734910249710083,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.6272653341293335,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 719
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 28.8,
+      "grad_norm": 6.140714645385742,
+      "kl": 1.9834489822387695,
+      "learning_rate": 3.3550503583141726e-06,
+      "loss": 0.0793,
+      "reward": -2.075000047683716,
+      "reward_std": 1.2867920398712158,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.675000011920929,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 720
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 28.84,
+      "grad_norm": 0.5307136178016663,
+      "kl": 0.8859113454818726,
+      "learning_rate": 3.3499230993274857e-06,
+      "loss": 0.0354,
+      "reward": 2.316016435623169,
+      "reward_std": 0.2834460735321045,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.4339835047721863,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 721
+    },
+    {
+      "completion_length": 613.0,
+      "epoch": 28.88,
+      "grad_norm": 0.3882230222225189,
+      "kl": 1.422294020652771,
+      "learning_rate": 3.344791795008318e-06,
+      "loss": 0.0569,
+      "reward": 1.4253931045532227,
+      "reward_std": 3.3321378231048584,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.6499999761581421,
+      "rewards/wrapped_driving_reward": -0.8496068120002747,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 722
+    },
+    {
+      "completion_length": 371.0,
+      "epoch": 28.92,
+      "grad_norm": 0.5566720366477966,
+      "kl": 0.9256033897399902,
+      "learning_rate": 3.339656469779856e-06,
+      "loss": 0.037,
+      "reward": 3.1350290775299072,
+      "reward_std": 0.2685109078884125,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.1350291520357132,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 723
+    },
+    {
+      "completion_length": 580.0,
+      "epoch": 28.96,
+      "grad_norm": 0.42029738426208496,
+      "kl": 0.6244791746139526,
+      "learning_rate": 3.3345171480844275e-06,
+      "loss": 0.025,
+      "reward": 3.0256998538970947,
+      "reward_std": 0.4103839099407196,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.025699838995933533,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 724
+    },
+    {
+      "completion_length": 596.0,
+      "epoch": 29.0,
+      "grad_norm": 0.5300978422164917,
+      "kl": 1.0864909887313843,
+      "learning_rate": 3.3293738543833807e-06,
+      "loss": 0.0435,
+      "reward": 1.6714462041854858,
+      "reward_std": 3.4748284816741943,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -0.7035538554191589,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 725
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 29.04,
+      "grad_norm": 0.5925276875495911,
+      "kl": 1.2335766553878784,
+      "learning_rate": 3.3242266131569685e-06,
+      "loss": 0.0493,
+      "reward": 1.1432609558105469,
+      "reward_std": 3.1195735931396484,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.2317389249801636,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 726
+    },
+    {
+      "completion_length": 623.0,
+      "epoch": 29.08,
+      "grad_norm": 0.4075430929660797,
+      "kl": 0.5598421692848206,
+      "learning_rate": 3.3190754489042343e-06,
+      "loss": 0.0224,
+      "reward": 2.513282537460327,
+      "reward_std": 0.3340396583080292,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.4867174029350281,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 727
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 29.12,
+      "grad_norm": 111.52384948730469,
+      "kl": 5.0521063804626465,
+      "learning_rate": 3.313920386142892e-06,
+      "loss": 0.2021,
+      "reward": 2.7720518112182617,
+      "reward_std": 0.9143410325050354,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.39705172181129456,
+      "rewards/wrapped_format_reward": 0.375,
+      "step": 728
+    },
+    {
+      "completion_length": 597.0,
+      "epoch": 29.16,
+      "grad_norm": 0.4811553955078125,
+      "kl": 1.3416039943695068,
+      "learning_rate": 3.308761449409213e-06,
+      "loss": 0.0537,
+      "reward": 3.2421016693115234,
+      "reward_std": 0.36916667222976685,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.367101788520813,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 729
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 29.2,
+      "grad_norm": 0.41855716705322266,
+      "kl": 1.2373117208480835,
+      "learning_rate": 3.303598663257904e-06,
+      "loss": 0.0495,
+      "reward": 2.902125835418701,
+      "reward_std": 0.21973761916160583,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.027125656604766846,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 730
+    },
+    {
+      "completion_length": 496.0,
+      "epoch": 29.24,
+      "grad_norm": 0.5248737931251526,
+      "kl": 1.0021467208862305,
+      "learning_rate": 3.298432052261998e-06,
+      "loss": 0.0401,
+      "reward": 3.6586220264434814,
+      "reward_std": 0.3303280174732208,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.6586220264434814,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 731
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 29.28,
+      "grad_norm": 0.4102158546447754,
+      "kl": 1.7108817100524902,
+      "learning_rate": 3.293261641012731e-06,
+      "loss": 0.0684,
+      "reward": 2.710984230041504,
+      "reward_std": 0.6047499775886536,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.28901582956314087,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 732
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 29.32,
+      "grad_norm": 0.4515760540962219,
+      "kl": 1.3170658349990845,
+      "learning_rate": 3.288087454119425e-06,
+      "loss": 0.0527,
+      "reward": -0.36018359661102295,
+      "reward_std": 3.6547343730926514,
+      "rewards/mpc_param_extraction_reward": 0.5,
+      "rewards/mpc_param_name_reward": 0.5,
+      "rewards/wrapped_driving_reward": -1.9851834774017334,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 733
+    },
+    {
+      "completion_length": 563.0,
+      "epoch": 29.36,
+      "grad_norm": 0.0380023717880249,
+      "kl": 0.8448745012283325,
+      "learning_rate": 3.282909516209374e-06,
+      "loss": 0.0338,
+      "reward": -1.0,
+      "reward_std": 0.0,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 734
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 29.4,
+      "grad_norm": 0.8251069188117981,
+      "kl": 1.3977208137512207,
+      "learning_rate": 3.277727851927727e-06,
+      "loss": 0.0559,
+      "reward": -1.3977272510528564,
+      "reward_std": 0.4886803925037384,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9772727489471436,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.625,
+      "step": 735
+    },
+    {
+      "completion_length": 639.0,
+      "epoch": 29.44,
+      "grad_norm": 0.5299381613731384,
+      "kl": 1.5736980438232422,
+      "learning_rate": 3.272542485937369e-06,
+      "loss": 0.0629,
+      "reward": 2.132500648498535,
+      "reward_std": 2.141465902328491,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.8674995303153992,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 736
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 29.48,
+      "grad_norm": 0.49962857365608215,
+      "kl": 0.6524366140365601,
+      "learning_rate": 3.2673534429188005e-06,
+      "loss": 0.0261,
+      "reward": 3.875,
+      "reward_std": 0.25,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 1.0,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 737
+    },
+    {
+      "completion_length": 573.0,
+      "epoch": 29.52,
+      "grad_norm": 0.3988845646381378,
+      "kl": 1.1060757637023926,
+      "learning_rate": 3.2621607475700272e-06,
+      "loss": 0.0442,
+      "reward": 3.0906763076782227,
+      "reward_std": 0.19022376835346222,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.09067624807357788,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 738
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 29.56,
+      "grad_norm": 0.59360671043396,
+      "kl": 1.5008355379104614,
+      "learning_rate": 3.256964424606437e-06,
+      "loss": 0.06,
+      "reward": -1.25,
+      "reward_std": 0.28867512941360474,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 739
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 29.6,
+      "grad_norm": 0.42812031507492065,
+      "kl": 0.4313167333602905,
+      "learning_rate": 3.2517644987606827e-06,
+      "loss": 0.0173,
+      "reward": 2.9479095935821533,
+      "reward_std": 0.5999106168746948,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.875,
+      "rewards/wrapped_driving_reward": 0.5729095935821533,
+      "rewards/wrapped_format_reward": 0.5,
+      "step": 740
+    },
+    {
+      "completion_length": 633.0,
+      "epoch": 29.64,
+      "grad_norm": 0.4350597858428955,
+      "kl": 1.3522546291351318,
+      "learning_rate": 3.2465609947825692e-06,
+      "loss": 0.0541,
+      "reward": 2.7952303886413574,
+      "reward_std": 0.5983152389526367,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9583333134651184,
+      "rewards/wrapped_driving_reward": -0.1631031036376953,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 741
+    },
+    {
+      "completion_length": 512.0,
+      "epoch": 29.68,
+      "grad_norm": 0.5423455238342285,
+      "kl": 0.5965325236320496,
+      "learning_rate": 3.2413539374389275e-06,
+      "loss": 0.0239,
+      "reward": 2.7277941703796387,
+      "reward_std": 0.38129425048828125,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.022205986082553864,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 742
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 29.72,
+      "grad_norm": 0.40301433205604553,
+      "kl": 1.3201624155044556,
+      "learning_rate": 3.2361433515135053e-06,
+      "loss": 0.0528,
+      "reward": 1.1487705707550049,
+      "reward_std": 3.1022140979766846,
+      "rewards/mpc_param_extraction_reward": 0.75,
+      "rewards/mpc_param_name_reward": 0.75,
+      "rewards/wrapped_driving_reward": -1.2262296676635742,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 743
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 29.76,
+      "grad_norm": 0.5065982341766357,
+      "kl": 1.7616549730300903,
+      "learning_rate": 3.230929261806842e-06,
+      "loss": 0.0705,
+      "reward": 2.8082611560821533,
+      "reward_std": 0.5932582020759583,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": 0.058261215686798096,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 744
+    },
+    {
+      "completion_length": 553.0,
+      "epoch": 29.8,
+      "grad_norm": 0.06308308243751526,
+      "kl": 1.113646388053894,
+      "learning_rate": 3.225711693136156e-06,
+      "loss": 0.0445,
+      "reward": -1.0,
+      "reward_std": 0.0,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 745
+    },
+    {
+      "completion_length": 546.0,
+      "epoch": 29.84,
+      "grad_norm": 0.4473098814487457,
+      "kl": 0.48006105422973633,
+      "learning_rate": 3.2204906703352236e-06,
+      "loss": 0.0192,
+      "reward": -1.125,
+      "reward_std": 0.25,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -4.0,
+      "rewards/wrapped_format_reward": 0.875,
+      "step": 746
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 29.88,
+      "grad_norm": 0.4587174654006958,
+      "kl": 1.3874768018722534,
+      "learning_rate": 3.215266218254261e-06,
+      "loss": 0.0555,
+      "reward": 2.485217809677124,
+      "reward_std": 1.0316221714019775,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 1.0,
+      "rewards/wrapped_driving_reward": -0.514782190322876,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 747
+    },
+    {
+      "completion_length": 460.0,
+      "epoch": 29.92,
+      "grad_norm": 0.47659605741500854,
+      "kl": 0.39824411273002625,
+      "learning_rate": 3.2100383617598075e-06,
+      "loss": 0.0159,
+      "reward": 2.951653480529785,
+      "reward_std": 0.28944987058639526,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9895833134651184,
+      "rewards/wrapped_driving_reward": -0.03792976588010788,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 748
+    },
+    {
+      "completion_length": 714.0,
+      "epoch": 29.96,
+      "grad_norm": 0.47844377160072327,
+      "kl": 1.2070376873016357,
+      "learning_rate": 3.2048071257346043e-06,
+      "loss": 0.0483,
+      "reward": 0.6153709888458252,
+      "reward_std": 1.9917418956756592,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9750000238418579,
+      "rewards/wrapped_driving_reward": -2.1096291542053223,
+      "rewards/wrapped_format_reward": 0.75,
+      "step": 749
+    },
+    {
+      "completion_length": 750.0,
+      "epoch": 30.0,
+      "grad_norm": 0.741182804107666,
+      "kl": 1.9630638360977173,
+      "learning_rate": 3.199572535077481e-06,
+      "loss": 0.0785,
+      "reward": 3.7558329105377197,
+      "reward_std": 0.19139595329761505,
+      "rewards/mpc_param_extraction_reward": 1.0,
+      "rewards/mpc_param_name_reward": 0.9608585834503174,
+      "rewards/wrapped_driving_reward": 0.7949742674827576,
+      "rewards/wrapped_format_reward": 1.0,
+      "step": 750
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 1600,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 64,
+  "save_steps": 250,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}