diff --git "a/comparison/trainer_state.json" "b/comparison/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/comparison/trainer_state.json"
@@ -0,0 +1,36433 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 4.16,
+  "eval_steps": 500,
+  "global_step": 2600,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 175.609375,
+      "epoch": 0.0016,
+      "grad_norm": 3.728504137385421,
+      "kl": 0.0,
+      "learning_rate": 9.998666666666665e-07,
+      "loss": -0.0,
+      "reward": 1.15625,
+      "reward_std": 0.4304669052362442,
+      "rewards/format_reward": 0.9375,
+      "rewards/score_reward": 0.21875,
+      "step": 1
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 174.734375,
+      "epoch": 0.0032,
+      "grad_norm": 3.5877728499458716,
+      "kl": 0.000766754150390625,
+      "learning_rate": 9.997333333333333e-07,
+      "loss": 0.0,
+      "reward": 1.203125,
+      "reward_std": 0.35103991627693176,
+      "rewards/format_reward": 0.96875,
+      "rewards/score_reward": 0.234375,
+      "step": 2
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 159.09375,
+      "epoch": 0.0048,
+      "grad_norm": 7.332286353609248,
+      "kl": 0.001434326171875,
+      "learning_rate": 9.996e-07,
+      "loss": 0.0,
+      "reward": 1.40625,
+      "reward_std": 0.5321738421916962,
+      "rewards/format_reward": 0.921875,
+      "rewards/score_reward": 0.484375,
+      "step": 3
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 158.578125,
+      "epoch": 0.0064,
+      "grad_norm": 5.429760827851981,
+      "kl": 0.001804351806640625,
+      "learning_rate": 9.994666666666665e-07,
+      "loss": 0.0,
+      "reward": 1.421875,
+      "reward_std": 0.32878512144088745,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.4375,
+      "step": 4
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 164.421875,
+      "epoch": 0.008,
+      "grad_norm": 5.77631127018886,
+      "kl": 0.00250244140625,
+      "learning_rate": 9.993333333333333e-07,
+      "loss": 0.0,
+      "reward": 1.609375,
+      "reward_std": 0.3661493957042694,
+      "rewards/format_reward": 0.96875,
+      "rewards/score_reward": 0.640625,
+      "step": 5
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 156.328125,
+      "epoch": 0.0096,
+      "grad_norm": 3.700751956484627,
+      "kl": 0.003173828125,
+      "learning_rate": 9.992e-07,
+      "loss": 0.0,
+      "reward": 1.484375,
+      "reward_std": 0.27883461862802505,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.5,
+      "step": 6
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 146.5625,
+      "epoch": 0.0112,
+      "grad_norm": 5.261480682221133,
+      "kl": 0.00312042236328125,
+      "learning_rate": 9.990666666666667e-07,
+      "loss": 0.0,
+      "reward": 1.53125,
+      "reward_std": 0.1767766885459423,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.53125,
+      "step": 7
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 151.84375,
+      "epoch": 0.0128,
+      "grad_norm": 7.5666474560467005,
+      "kl": 0.041748046875,
+      "learning_rate": 9.989333333333333e-07,
+      "loss": 0.0,
+      "reward": 1.578125,
+      "reward_std": 0.31761831045150757,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.59375,
+      "step": 8
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 143.046875,
+      "epoch": 0.0144,
+      "grad_norm": 4.784702176594941,
+      "kl": 0.0045928955078125,
+      "learning_rate": 9.988e-07,
+      "loss": 0.0,
+      "reward": 1.640625,
+      "reward_std": 0.3265698626637459,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.65625,
+      "step": 9
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 142.15625,
+      "epoch": 0.016,
+      "grad_norm": 5.309552099226207,
+      "kl": 0.00579833984375,
+      "learning_rate": 9.986666666666667e-07,
+      "loss": 0.0,
+      "reward": 1.25,
+      "reward_std": 0.2177756428718567,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.265625,
+      "step": 10
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.640625,
+      "epoch": 0.0176,
+      "grad_norm": 4.972066628407755,
+      "kl": 0.0072174072265625,
+      "learning_rate": 9.985333333333332e-07,
+      "loss": 0.0,
+      "reward": 1.421875,
+      "reward_std": 0.10205793380737305,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.421875,
+      "step": 11
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.359375,
+      "epoch": 0.0192,
+      "grad_norm": 7.66404216328786,
+      "kl": 0.0086212158203125,
+      "learning_rate": 9.983999999999998e-07,
+      "loss": 0.0,
+      "reward": 1.5625,
+      "reward_std": 0.2756393998861313,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5625,
+      "step": 12
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.484375,
+      "epoch": 0.0208,
+      "grad_norm": 7.733346302291279,
+      "kl": 0.010528564453125,
+      "learning_rate": 9.982666666666666e-07,
+      "loss": 0.0,
+      "reward": 1.5625,
+      "reward_std": 0.3029785603284836,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5625,
+      "step": 13
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.921875,
+      "epoch": 0.0224,
+      "grad_norm": 10.285684389687534,
+      "kl": 0.011932373046875,
+      "learning_rate": 9.981333333333332e-07,
+      "loss": 0.0,
+      "reward": 1.421875,
+      "reward_std": 0.2382849156856537,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.421875,
+      "step": 14
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.765625,
+      "epoch": 0.024,
+      "grad_norm": 7.871815334292557,
+      "kl": 0.014556884765625,
+      "learning_rate": 9.98e-07,
+      "loss": 0.0,
+      "reward": 1.421875,
+      "reward_std": 0.2109457477927208,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.421875,
+      "step": 15
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.140625,
+      "epoch": 0.0256,
+      "grad_norm": 9.427194965002142,
+      "kl": 0.01519775390625,
+      "learning_rate": 9.978666666666666e-07,
+      "loss": 0.0,
+      "reward": 1.3125,
+      "reward_std": 0.22461533546447754,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.3125,
+      "step": 16
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.953125,
+      "epoch": 0.0272,
+      "grad_norm": 6.119096906142894,
+      "kl": 0.01727294921875,
+      "learning_rate": 9.977333333333334e-07,
+      "loss": 0.0,
+      "reward": 1.703125,
+      "reward_std": 0.28930898010730743,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.71875,
+      "step": 17
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.234375,
+      "epoch": 0.0288,
+      "grad_norm": 13.941143411441205,
+      "kl": 0.01800537109375,
+      "learning_rate": 9.976e-07,
+      "loss": 0.0,
+      "reward": 1.578125,
+      "reward_std": 0.19408093392848969,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.578125,
+      "step": 18
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.046875,
+      "epoch": 0.0304,
+      "grad_norm": 5.569779646635424,
+      "kl": 0.02099609375,
+      "learning_rate": 9.974666666666666e-07,
+      "loss": 0.0,
+      "reward": 1.484375,
+      "reward_std": 0.2993340939283371,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.484375,
+      "step": 19
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.28125,
+      "epoch": 0.032,
+      "grad_norm": 6.532023446264424,
+      "kl": 0.0250244140625,
+      "learning_rate": 9.973333333333332e-07,
+      "loss": 0.0,
+      "reward": 1.640625,
+      "reward_std": 0.282469280064106,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.640625,
+      "step": 20
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.421875,
+      "epoch": 0.0336,
+      "grad_norm": 10.48038986529557,
+      "kl": 0.0220947265625,
+      "learning_rate": 9.972e-07,
+      "loss": 0.0,
+      "reward": 1.4375,
+      "reward_std": 0.23356688022613525,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.4375,
+      "step": 21
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.625,
+      "epoch": 0.0352,
+      "grad_norm": 11.249157393157311,
+      "kl": 0.0244140625,
+      "learning_rate": 9.970666666666665e-07,
+      "loss": 0.0,
+      "reward": 1.5,
+      "reward_std": 0.23356688767671585,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 22
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.359375,
+      "epoch": 0.0368,
+      "grad_norm": 11.121440868644864,
+      "kl": 0.02313232421875,
+      "learning_rate": 9.969333333333333e-07,
+      "loss": 0.0,
+      "reward": 1.359375,
+      "reward_std": 0.16887323930859566,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.359375,
+      "step": 23
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.078125,
+      "epoch": 0.0384,
+      "grad_norm": 10.007188099144804,
+      "kl": 0.02490234375,
+      "learning_rate": 9.968e-07,
+      "loss": 0.0,
+      "reward": 1.765625,
+      "reward_std": 0.23568854480981827,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.765625,
+      "step": 24
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.328125,
+      "epoch": 0.04,
+      "grad_norm": 6.6350782610438666,
+      "kl": 0.02679443359375,
+      "learning_rate": 9.966666666666667e-07,
+      "loss": 0.0,
+      "reward": 1.6875,
+      "reward_std": 0.1157275140285492,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.6875,
+      "step": 25
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.484375,
+      "epoch": 0.0416,
+      "grad_norm": 5.442446744311865,
+      "kl": 0.024658203125,
+      "learning_rate": 9.965333333333333e-07,
+      "loss": 0.0,
+      "reward": 1.640625,
+      "reward_std": 0.17358146607875824,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.640625,
+      "step": 26
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.40625,
+      "epoch": 0.0432,
+      "grad_norm": 3.4648891323738895,
+      "kl": 0.0286865234375,
+      "learning_rate": 9.964e-07,
+      "loss": 0.0,
+      "reward": 1.453125,
+      "reward_std": 0.23144522309303284,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.453125,
+      "step": 27
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.46875,
+      "epoch": 0.0448,
+      "grad_norm": 5.828663606532387,
+      "kl": 0.03369140625,
+      "learning_rate": 9.962666666666667e-07,
+      "loss": 0.0,
+      "reward": 1.578125,
+      "reward_std": 0.3845370262861252,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.578125,
+      "step": 28
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.109375,
+      "epoch": 0.0464,
+      "grad_norm": 9.009973731020866,
+      "kl": 0.0321044921875,
+      "learning_rate": 9.961333333333333e-07,
+      "loss": 0.0,
+      "reward": 1.40625,
+      "reward_std": 0.22201896458864212,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.40625,
+      "step": 29
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.0625,
+      "epoch": 0.048,
+      "grad_norm": 10.802013895253339,
+      "kl": 0.0347900390625,
+      "learning_rate": 9.959999999999999e-07,
+      "loss": 0.0,
+      "reward": 1.671875,
+      "reward_std": 0.32878512144088745,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.671875,
+      "step": 30
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.421875,
+      "epoch": 0.0496,
+      "grad_norm": 3.523692822749765,
+      "kl": 0.0386962890625,
+      "learning_rate": 9.958666666666667e-07,
+      "loss": 0.0,
+      "reward": 1.375,
+      "reward_std": 0.16675157845020294,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 31
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.078125,
+      "epoch": 0.0512,
+      "grad_norm": 4.325877820098955,
+      "kl": 0.0340576171875,
+      "learning_rate": 9.957333333333332e-07,
+      "loss": 0.0,
+      "reward": 1.71875,
+      "reward_std": 0.23356688022613525,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.71875,
+      "step": 32
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.625,
+      "epoch": 0.0528,
+      "grad_norm": 14.045123660002453,
+      "kl": 0.0361328125,
+      "learning_rate": 9.956e-07,
+      "loss": 0.0,
+      "reward": 1.484375,
+      "reward_std": 0.260896235704422,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.484375,
+      "step": 33
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.53125,
+      "epoch": 0.0544,
+      "grad_norm": 7.4319675518313,
+      "kl": 0.0406494140625,
+      "learning_rate": 9.954666666666666e-07,
+      "loss": 0.0,
+      "reward": 1.28125,
+      "reward_std": 0.1767766885459423,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.296875,
+      "step": 34
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.78125,
+      "epoch": 0.056,
+      "grad_norm": 12.77009725984137,
+      "kl": 0.0396728515625,
+      "learning_rate": 9.953333333333332e-07,
+      "loss": 0.0,
+      "reward": 1.671875,
+      "reward_std": 0.12255740165710449,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.671875,
+      "step": 35
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 101.734375,
+      "epoch": 0.0576,
+      "grad_norm": 6.388215508572979,
+      "kl": 0.0413818359375,
+      "learning_rate": 9.952e-07,
+      "loss": 0.0,
+      "reward": 1.59375,
+      "reward_std": 0.2619796246290207,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.59375,
+      "step": 36
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 99.359375,
+      "epoch": 0.0592,
+      "grad_norm": 26.240967421315954,
+      "kl": 0.045654296875,
+      "learning_rate": 9.950666666666666e-07,
+      "loss": 0.0,
+      "reward": 1.625,
+      "reward_std": 0.29143063724040985,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 37
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 98.703125,
+      "epoch": 0.0608,
+      "grad_norm": 5.795696665279101,
+      "kl": 0.04296875,
+      "learning_rate": 9.949333333333332e-07,
+      "loss": 0.0,
+      "reward": 1.5,
+      "reward_std": 0.22201895713806152,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 38
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 103.75,
+      "epoch": 0.0624,
+      "grad_norm": 5.422975019260023,
+      "kl": 0.04931640625,
+      "learning_rate": 9.948e-07,
+      "loss": 0.0,
+      "reward": 1.625,
+      "reward_std": 0.1462521068751812,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 39
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 101.953125,
+      "epoch": 0.064,
+      "grad_norm": 16.02356150358374,
+      "kl": 0.0491943359375,
+      "learning_rate": 9.946666666666666e-07,
+      "loss": 0.0,
+      "reward": 1.375,
+      "reward_std": 0.35400262475013733,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 40
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 100.421875,
+      "epoch": 0.0656,
+      "grad_norm": 62.44009659710585,
+      "kl": 0.050048828125,
+      "learning_rate": 9.945333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.640625,
+      "reward_std": 0.3234682232141495,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.640625,
+      "step": 41
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 102.328125,
+      "epoch": 0.0672,
+      "grad_norm": 7.855764423526861,
+      "kl": 0.0474853515625,
+      "learning_rate": 9.944e-07,
+      "loss": 0.0,
+      "reward": 1.53125,
+      "reward_std": 0.22201896458864212,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.53125,
+      "step": 42
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 99.140625,
+      "epoch": 0.0688,
+      "grad_norm": 5.5466323355184635,
+      "kl": 0.0555419921875,
+      "learning_rate": 9.942666666666665e-07,
+      "loss": 0.0001,
+      "reward": 1.609375,
+      "reward_std": 0.21778544783592224,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.609375,
+      "step": 43
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 97.234375,
+      "epoch": 0.0704,
+      "grad_norm": 3.8823598771109245,
+      "kl": 0.06005859375,
+      "learning_rate": 9.941333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.359375,
+      "reward_std": 0.1315089538693428,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.359375,
+      "step": 44
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 99.390625,
+      "epoch": 0.072,
+      "grad_norm": 8.67076806341889,
+      "kl": 0.0572509765625,
+      "learning_rate": 9.94e-07,
+      "loss": 0.0001,
+      "reward": 1.4375,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.4375,
+      "step": 45
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 96.546875,
+      "epoch": 0.0736,
+      "grad_norm": 4.07269905108586,
+      "kl": 0.075927734375,
+      "learning_rate": 9.938666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.53125,
+      "reward_std": 0.1462521068751812,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.53125,
+      "step": 46
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 93.765625,
+      "epoch": 0.0752,
+      "grad_norm": 5.873871020365467,
+      "kl": 0.05859375,
+      "learning_rate": 9.937333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.484375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.484375,
+      "step": 47
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 98.859375,
+      "epoch": 0.0768,
+      "grad_norm": 70.95519715989934,
+      "kl": 0.060791015625,
+      "learning_rate": 9.936e-07,
+      "loss": 0.0001,
+      "reward": 1.640625,
+      "reward_std": 0.16887323930859566,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.640625,
+      "step": 48
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 95.828125,
+      "epoch": 0.0784,
+      "grad_norm": 4.132366867885681,
+      "kl": 0.064697265625,
+      "learning_rate": 9.934666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.5625,
+      "reward_std": 0.10888781771063805,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5625,
+      "step": 49
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 92.109375,
+      "epoch": 0.08,
+      "grad_norm": 8.37469838018613,
+      "kl": 0.068115234375,
+      "learning_rate": 9.933333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.609375,
+      "reward_std": 0.1315089538693428,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.609375,
+      "step": 50
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 98.46875,
+      "epoch": 0.0816,
+      "grad_norm": 5.422655410387628,
+      "kl": 0.06640625,
+      "learning_rate": 9.931999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.53125,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.53125,
+      "step": 51
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 101.859375,
+      "epoch": 0.0832,
+      "grad_norm": 6.049109740335134,
+      "kl": 0.06640625,
+      "learning_rate": 9.930666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.53125,
+      "reward_std": 0.24511480331420898,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.53125,
+      "step": 52
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 92.109375,
+      "epoch": 0.0848,
+      "grad_norm": 2.971639698230816,
+      "kl": 0.077392578125,
+      "learning_rate": 9.929333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.46875,
+      "reward_std": 0.0883883461356163,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.46875,
+      "step": 53
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 93.78125,
+      "epoch": 0.0864,
+      "grad_norm": 5.857270191205943,
+      "kl": 0.076416015625,
+      "learning_rate": 9.928e-07,
+      "loss": 0.0001,
+      "reward": 1.65625,
+      "reward_std": 0.10888781771063805,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.65625,
+      "step": 54
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 96.890625,
+      "epoch": 0.088,
+      "grad_norm": 2.902835730212626,
+      "kl": 0.078125,
+      "learning_rate": 9.926666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.71875,
+      "reward_std": 0.0883883461356163,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.71875,
+      "step": 55
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 93.875,
+      "epoch": 0.0896,
+      "grad_norm": 4.769519782018857,
+      "kl": 0.075439453125,
+      "learning_rate": 9.925333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.640625,
+      "reward_std": 0.17782479152083397,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.640625,
+      "step": 56
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 91.828125,
+      "epoch": 0.0912,
+      "grad_norm": 5.742418747960446,
+      "kl": 0.081787109375,
+      "learning_rate": 9.923999999999998e-07,
+      "loss": 0.0001,
+      "reward": 1.5625,
+      "reward_std": 0.16675157472491264,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.578125,
+      "step": 57
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 88.484375,
+      "epoch": 0.0928,
+      "grad_norm": 2.128476259241305,
+      "kl": 0.090087890625,
+      "learning_rate": 9.922666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.65625,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.65625,
+      "step": 58
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 90.40625,
+      "epoch": 0.0944,
+      "grad_norm": 3.6944991922025943,
+      "kl": 0.08349609375,
+      "learning_rate": 9.921333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.515625,
+      "reward_std": 0.10205793008208275,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.53125,
+      "step": 59
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 88.84375,
+      "epoch": 0.096,
+      "grad_norm": 6.104593058583627,
+      "kl": 0.0888671875,
+      "learning_rate": 9.92e-07,
+      "loss": 0.0001,
+      "reward": 1.65625,
+      "reward_std": 0.10888782143592834,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.65625,
+      "step": 60
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 92.703125,
+      "epoch": 0.0976,
+      "grad_norm": 5.513507457668777,
+      "kl": 0.105712890625,
+      "learning_rate": 9.918666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.421875,
+      "reward_std": 0.1804211586713791,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.421875,
+      "step": 61
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 89.0,
+      "epoch": 0.0992,
+      "grad_norm": 3.7946899726309407,
+      "kl": 0.100341796875,
+      "learning_rate": 9.917333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.484375,
+      "reward_std": 0.12255740165710449,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.484375,
+      "step": 62
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 87.390625,
+      "epoch": 0.1008,
+      "grad_norm": 46.55597432708147,
+      "kl": 0.107177734375,
+      "learning_rate": 9.916e-07,
+      "loss": 0.0001,
+      "reward": 1.328125,
+      "reward_std": 0.11100947856903076,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.328125,
+      "step": 63
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 89.8125,
+      "epoch": 0.1024,
+      "grad_norm": 5.258022104689105,
+      "kl": 0.1044921875,
+      "learning_rate": 9.914666666666668e-07,
+      "loss": 0.0001,
+      "reward": 1.65625,
+      "reward_std": 0.1246790662407875,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.65625,
+      "step": 64
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 84.453125,
+      "epoch": 0.104,
+      "grad_norm": 9.473980767976478,
+      "kl": 0.095458984375,
+      "learning_rate": 9.913333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0883883461356163,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 65
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 86.265625,
+      "epoch": 0.1056,
+      "grad_norm": 2.8355192742259976,
+      "kl": 0.099853515625,
+      "learning_rate": 9.912e-07,
+      "loss": 0.0001,
+      "reward": 1.5625,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5625,
+      "step": 66
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 88.65625,
+      "epoch": 0.1072,
+      "grad_norm": 5.670795674167328,
+      "kl": 0.102294921875,
+      "learning_rate": 9.910666666666665e-07,
+      "loss": 0.0001,
+      "reward": 1.484375,
+      "reward_std": 0.10205793008208275,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.484375,
+      "step": 67
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 91.6875,
+      "epoch": 0.1088,
+      "grad_norm": 2.650427372720894,
+      "kl": 0.10888671875,
+      "learning_rate": 9.909333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.359375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.359375,
+      "step": 68
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 90.625,
+      "epoch": 0.1104,
+      "grad_norm": 4.255372483827343,
+      "kl": 0.100830078125,
+      "learning_rate": 9.908e-07,
+      "loss": 0.0001,
+      "reward": 1.84375,
+      "reward_std": 0.0883883461356163,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.84375,
+      "step": 69
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 91.640625,
+      "epoch": 0.112,
+      "grad_norm": 0.012472477689348649,
+      "kl": 0.10400390625,
+      "learning_rate": 9.906666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 70
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 86.03125,
+      "epoch": 0.1136,
+      "grad_norm": 19.129658602217194,
+      "kl": 0.11181640625,
+      "learning_rate": 9.905333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.90625,
+      "reward_std": 0.19727616757154465,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.90625,
+      "step": 71
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 90.09375,
+      "epoch": 0.1152,
+      "grad_norm": 7.564359017740217,
+      "kl": 0.123291015625,
+      "learning_rate": 9.903999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.703125,
+      "reward_std": 0.10205793380737305,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.703125,
+      "step": 72
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 98.484375,
+      "epoch": 0.1168,
+      "grad_norm": 3.7820003468021253,
+      "kl": 0.127197265625,
+      "learning_rate": 9.902666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.765625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.765625,
+      "step": 73
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 90.171875,
+      "epoch": 0.1184,
+      "grad_norm": 0.020297208549746548,
+      "kl": 0.115478515625,
+      "learning_rate": 9.901333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 74
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 90.734375,
+      "epoch": 0.12,
+      "grad_norm": 5.790890335136487,
+      "kl": 0.1083984375,
+      "learning_rate": 9.9e-07,
+      "loss": 0.0001,
+      "reward": 1.484375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.484375,
+      "step": 75
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 93.5,
+      "epoch": 0.1216,
+      "grad_norm": 10.702064861688259,
+      "kl": 0.1044921875,
+      "learning_rate": 9.898666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.71875,
+      "reward_std": 0.22461533173918724,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.71875,
+      "step": 76
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 92.203125,
+      "epoch": 0.1232,
+      "grad_norm": 6.75562123463835,
+      "kl": 0.112060546875,
+      "learning_rate": 9.897333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.328125,
+      "reward_std": 0.1530819833278656,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.328125,
+      "step": 77
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 91.5,
+      "epoch": 0.1248,
+      "grad_norm": 3.148246606154765,
+      "kl": 0.107666015625,
+      "learning_rate": 9.896e-07,
+      "loss": 0.0001,
+      "reward": 1.171875,
+      "reward_std": 0.10205793008208275,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.171875,
+      "step": 78
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 93.796875,
+      "epoch": 0.1264,
+      "grad_norm": 30.511224181864943,
+      "kl": 0.114990234375,
+      "learning_rate": 9.894666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 79
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 92.203125,
+      "epoch": 0.128,
+      "grad_norm": 0.011425677610074382,
+      "kl": 0.10107421875,
+      "learning_rate": 9.893333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 80
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 94.03125,
+      "epoch": 0.1296,
+      "grad_norm": 3.7439305241665277,
+      "kl": 0.102294921875,
+      "learning_rate": 9.892e-07,
+      "loss": 0.0001,
+      "reward": 1.546875,
+      "reward_std": 0.10205793008208275,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.546875,
+      "step": 81
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 94.578125,
+      "epoch": 0.1312,
+      "grad_norm": 5.654825318326709,
+      "kl": 0.096435546875,
+      "learning_rate": 9.890666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.8125,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.8125,
+      "step": 82
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 92.0625,
+      "epoch": 0.1328,
+      "grad_norm": 9.006952492349495,
+      "kl": 0.1103515625,
+      "learning_rate": 9.889333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.609375,
+      "reward_std": 0.17358146607875824,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.609375,
+      "step": 83
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 101.390625,
+      "epoch": 0.1344,
+      "grad_norm": 7.109716605903811,
+      "kl": 0.119140625,
+      "learning_rate": 9.888e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0883883461356163,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 84
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 98.8125,
+      "epoch": 0.136,
+      "grad_norm": 3.6011529882949644,
+      "kl": 0.104248046875,
+      "learning_rate": 9.886666666666665e-07,
+      "loss": 0.0001,
+      "reward": 1.828125,
+      "reward_std": 0.10205793380737305,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.828125,
+      "step": 85
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 96.15625,
+      "epoch": 0.1376,
+      "grad_norm": 6.30542637305878,
+      "kl": 0.11376953125,
+      "learning_rate": 9.885333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.421875,
+      "reward_std": 0.16887323558330536,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.421875,
+      "step": 86
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 99.5,
+      "epoch": 0.1392,
+      "grad_norm": 8.219421350554818,
+      "kl": 0.10546875,
+      "learning_rate": 9.884e-07,
+      "loss": 0.0001,
+      "reward": 1.5625,
+      "reward_std": 0.1552036553621292,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5625,
+      "step": 87
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 99.671875,
+      "epoch": 0.1408,
+      "grad_norm": 5.789913534597948,
+      "kl": 0.103515625,
+      "learning_rate": 9.882666666666665e-07,
+      "loss": 0.0001,
+      "reward": 1.546875,
+      "reward_std": 0.1530819833278656,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.546875,
+      "step": 88
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 102.21875,
+      "epoch": 0.1424,
+      "grad_norm": 6.124615970787052,
+      "kl": 0.102294921875,
+      "learning_rate": 9.881333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.22461533546447754,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 89
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 99.390625,
+      "epoch": 0.144,
+      "grad_norm": 3.368336174475924,
+      "kl": 0.1064453125,
+      "learning_rate": 9.88e-07,
+      "loss": 0.0001,
+      "reward": 1.8125,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.8125,
+      "step": 90
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 102.0625,
+      "epoch": 0.1456,
+      "grad_norm": 3.716273435909961,
+      "kl": 0.104736328125,
+      "learning_rate": 9.878666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.640625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.640625,
+      "step": 91
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 103.109375,
+      "epoch": 0.1472,
+      "grad_norm": 3.8452788103732893,
+      "kl": 0.112060546875,
+      "learning_rate": 9.877333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.40625,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.40625,
+      "step": 92
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 99.234375,
+      "epoch": 0.1488,
+      "grad_norm": 8.304133007098699,
+      "kl": 0.116943359375,
+      "learning_rate": 9.876e-07,
+      "loss": 0.0001,
+      "reward": 1.265625,
+      "reward_std": 0.2109457515180111,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.265625,
+      "step": 93
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 93.171875,
+      "epoch": 0.1504,
+      "grad_norm": 8.189957962691532,
+      "kl": 0.11767578125,
+      "learning_rate": 9.874666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.6875,
+      "reward_std": 0.10888782143592834,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.6875,
+      "step": 94
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 94.046875,
+      "epoch": 0.152,
+      "grad_norm": 19.069487658900236,
+      "kl": 0.12158203125,
+      "learning_rate": 9.873333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.3125,
+      "reward_std": 0.1462521068751812,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.3125,
+      "step": 95
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 94.328125,
+      "epoch": 0.1536,
+      "grad_norm": 9.633706677553036,
+      "kl": 0.133056640625,
+      "learning_rate": 9.871999999999998e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.30038218945264816,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 96
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 95.734375,
+      "epoch": 0.1552,
+      "grad_norm": 2.2900788032850286,
+      "kl": 0.113037109375,
+      "learning_rate": 9.870666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.65625,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.65625,
+      "step": 97
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 96.125,
+      "epoch": 0.1568,
+      "grad_norm": 6.308447355803907,
+      "kl": 0.1357421875,
+      "learning_rate": 9.869333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.296875,
+      "reward_std": 0.19939782470464706,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.296875,
+      "step": 98
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 98.28125,
+      "epoch": 0.1584,
+      "grad_norm": 3.5523775678870853,
+      "kl": 0.11279296875,
+      "learning_rate": 9.868e-07,
+      "loss": 0.0001,
+      "reward": 1.5625,
+      "reward_std": 0.10888781771063805,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5625,
+      "step": 99
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 95.953125,
+      "epoch": 0.16,
+      "grad_norm": 5.317298185634635,
+      "kl": 0.116943359375,
+      "learning_rate": 9.866666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.578125,
+      "reward_std": 0.11100947856903076,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.578125,
+      "step": 100
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 95.890625,
+      "epoch": 0.1616,
+      "grad_norm": 27.002042200000048,
+      "kl": 0.1123046875,
+      "learning_rate": 9.865333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.765625,
+      "reward_std": 0.16887325048446655,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.765625,
+      "step": 101
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 98.796875,
+      "epoch": 0.1632,
+      "grad_norm": 9.429944433706737,
+      "kl": 0.1298828125,
+      "learning_rate": 9.864e-07,
+      "loss": 0.0001,
+      "reward": 1.53125,
+      "reward_std": 0.1462521068751812,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.53125,
+      "step": 102
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 92.578125,
+      "epoch": 0.1648,
+      "grad_norm": 5.314286327837528,
+      "kl": 0.12158203125,
+      "learning_rate": 9.862666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.515625,
+      "reward_std": 0.12255740165710449,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.515625,
+      "step": 103
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 96.265625,
+      "epoch": 0.1664,
+      "grad_norm": 4.313430029263258,
+      "kl": 0.13818359375,
+      "learning_rate": 9.861333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.4375,
+      "reward_std": 0.1157275140285492,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.4375,
+      "step": 104
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 97.015625,
+      "epoch": 0.168,
+      "grad_norm": 4.094172800704466,
+      "kl": 0.13525390625,
+      "learning_rate": 9.86e-07,
+      "loss": 0.0001,
+      "reward": 1.453125,
+      "reward_std": 0.11100947856903076,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.453125,
+      "step": 105
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 92.140625,
+      "epoch": 0.1696,
+      "grad_norm": 6.703599379994853,
+      "kl": 0.16455078125,
+      "learning_rate": 9.858666666666665e-07,
+      "loss": 0.0002,
+      "reward": 1.484375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.484375,
+      "step": 106
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 98.15625,
+      "epoch": 0.1712,
+      "grad_norm": 3.596921121025124,
+      "kl": 0.135009765625,
+      "learning_rate": 9.857333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.609375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.609375,
+      "step": 107
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 98.25,
+      "epoch": 0.1728,
+      "grad_norm": 11.858633877972297,
+      "kl": 0.130126953125,
+      "learning_rate": 9.856e-07,
+      "loss": 0.0001,
+      "reward": 1.734375,
+      "reward_std": 0.23144522309303284,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 108
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 91.609375,
+      "epoch": 0.1744,
+      "grad_norm": 7.05901946909446,
+      "kl": 0.134765625,
+      "learning_rate": 9.854666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.16675157845020294,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 109
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 93.40625,
+      "epoch": 0.176,
+      "grad_norm": 9.602483569882514,
+      "kl": 0.12158203125,
+      "learning_rate": 9.853333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.609375,
+      "reward_std": 0.16887324303388596,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.609375,
+      "step": 110
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 93.921875,
+      "epoch": 0.1776,
+      "grad_norm": 1.5753957906367533,
+      "kl": 0.110595703125,
+      "learning_rate": 9.852e-07,
+      "loss": 0.0001,
+      "reward": 1.59375,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.59375,
+      "step": 111
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 90.578125,
+      "epoch": 0.1792,
+      "grad_norm": 11.780737191829196,
+      "kl": 0.127685546875,
+      "learning_rate": 9.850666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.515625,
+      "reward_std": 0.12255740165710449,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.515625,
+      "step": 112
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 92.5625,
+      "epoch": 0.1808,
+      "grad_norm": 14.447711266054187,
+      "kl": 0.607421875,
+      "learning_rate": 9.849333333333333e-07,
+      "loss": 0.0006,
+      "reward": 1.75,
+      "reward_std": 0.1825428232550621,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 113
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 91.8125,
+      "epoch": 0.1824,
+      "grad_norm": 10.712765007883347,
+      "kl": 0.14453125,
+      "learning_rate": 9.847999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.65625,
+      "reward_std": 0.1462521068751812,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.65625,
+      "step": 114
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 90.125,
+      "epoch": 0.184,
+      "grad_norm": 3.125224471420021,
+      "kl": 0.13427734375,
+      "learning_rate": 9.846666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.515625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.515625,
+      "step": 115
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 92.265625,
+      "epoch": 0.1856,
+      "grad_norm": 2.670762496579475,
+      "kl": 0.14892578125,
+      "learning_rate": 9.845333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.59375,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.59375,
+      "step": 116
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 86.921875,
+      "epoch": 0.1872,
+      "grad_norm": 0.01724566644676974,
+      "kl": 0.125244140625,
+      "learning_rate": 9.844e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 117
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 92.875,
+      "epoch": 0.1888,
+      "grad_norm": 4.92436301103278,
+      "kl": 0.130859375,
+      "learning_rate": 9.842666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.65625,
+      "reward_std": 0.1246790662407875,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.65625,
+      "step": 118
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 95.375,
+      "epoch": 0.1904,
+      "grad_norm": 7.892677939836195,
+      "kl": 0.12353515625,
+      "learning_rate": 9.841333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.765625,
+      "reward_std": 0.12255740165710449,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.765625,
+      "step": 119
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 89.234375,
+      "epoch": 0.192,
+      "grad_norm": 9.121719076868105,
+      "kl": 0.134033203125,
+      "learning_rate": 9.84e-07,
+      "loss": 0.0001,
+      "reward": 1.484375,
+      "reward_std": 0.1315089538693428,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.484375,
+      "step": 120
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 93.0,
+      "epoch": 0.1936,
+      "grad_norm": 13.50840822511036,
+      "kl": 0.126220703125,
+      "learning_rate": 9.838666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.40625,
+      "reward_std": 0.1552036553621292,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.40625,
+      "step": 121
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 87.859375,
+      "epoch": 0.1952,
+      "grad_norm": 0.03187120367106415,
+      "kl": 0.126708984375,
+      "learning_rate": 9.837333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 122
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 87.0625,
+      "epoch": 0.1968,
+      "grad_norm": 7.273851460790811,
+      "kl": 0.130615234375,
+      "learning_rate": 9.836e-07,
+      "loss": 0.0001,
+      "reward": 1.734375,
+      "reward_std": 0.10205793008208275,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 123
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 91.296875,
+      "epoch": 0.1984,
+      "grad_norm": 7.619634448634981,
+      "kl": 0.142822265625,
+      "learning_rate": 9.834666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.78125,
+      "reward_std": 0.1293872892856598,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.78125,
+      "step": 124
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 89.1875,
+      "epoch": 0.2,
+      "grad_norm": 8.988254476248398,
+      "kl": 0.12744140625,
+      "learning_rate": 9.833333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.53125,
+      "reward_std": 0.1462521068751812,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.53125,
+      "step": 125
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 92.90625,
+      "epoch": 0.2016,
+      "grad_norm": 6.161176951547785,
+      "kl": 0.119873046875,
+      "learning_rate": 9.832e-07,
+      "loss": 0.0001,
+      "reward": 1.65625,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.65625,
+      "step": 126
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 88.0,
+      "epoch": 0.2032,
+      "grad_norm": 0.01684731233548339,
+      "kl": 0.118896484375,
+      "learning_rate": 9.830666666666665e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 127
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 89.5,
+      "epoch": 0.2048,
+      "grad_norm": 5.102919185043143,
+      "kl": 0.13134765625,
+      "learning_rate": 9.829333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 128
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 89.609375,
+      "epoch": 0.2064,
+      "grad_norm": 6.855300225416732,
+      "kl": 0.13134765625,
+      "learning_rate": 9.828e-07,
+      "loss": 0.0001,
+      "reward": 1.5625,
+      "reward_std": 0.1157275140285492,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5625,
+      "step": 129
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 89.0,
+      "epoch": 0.208,
+      "grad_norm": 6.065550624512237,
+      "kl": 0.1279296875,
+      "learning_rate": 9.826666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.1293872892856598,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 130
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 90.765625,
+      "epoch": 0.2096,
+      "grad_norm": 11.946741073062606,
+      "kl": 1.091796875,
+      "learning_rate": 9.825333333333333e-07,
+      "loss": 0.0011,
+      "reward": 1.234375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.234375,
+      "step": 131
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 89.765625,
+      "epoch": 0.2112,
+      "grad_norm": 3.400938428286766,
+      "kl": 0.1357421875,
+      "learning_rate": 9.824e-07,
+      "loss": 0.0001,
+      "reward": 1.796875,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.796875,
+      "step": 132
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 92.328125,
+      "epoch": 0.2128,
+      "grad_norm": 2.020901131896129,
+      "kl": 0.146240234375,
+      "learning_rate": 9.822666666666665e-07,
+      "loss": 0.0001,
+      "reward": 1.515625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.515625,
+      "step": 133
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 88.390625,
+      "epoch": 0.2144,
+      "grad_norm": 4.409468205871754,
+      "kl": 0.1376953125,
+      "learning_rate": 9.821333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.4375,
+      "reward_std": 0.10888781771063805,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.4375,
+      "step": 134
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 88.5625,
+      "epoch": 0.216,
+      "grad_norm": 0.018562755177581852,
+      "kl": 0.1337890625,
+      "learning_rate": 9.819999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 135
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 90.796875,
+      "epoch": 0.2176,
+      "grad_norm": 5.225532756848387,
+      "kl": 0.126708984375,
+      "learning_rate": 9.818666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.671875,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.671875,
+      "step": 136
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 90.359375,
+      "epoch": 0.2192,
+      "grad_norm": 1.4993585656268673,
+      "kl": 0.12939453125,
+      "learning_rate": 9.817333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.859375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.859375,
+      "step": 137
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 91.625,
+      "epoch": 0.2208,
+      "grad_norm": 0.02131959296280112,
+      "kl": 0.1328125,
+      "learning_rate": 9.816e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 138
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 91.09375,
+      "epoch": 0.2224,
+      "grad_norm": 6.043605546540833,
+      "kl": 0.154296875,
+      "learning_rate": 9.814666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.734375,
+      "reward_std": 0.1315089464187622,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 139
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 84.828125,
+      "epoch": 0.224,
+      "grad_norm": 5.198102322189676,
+      "kl": 0.16259765625,
+      "learning_rate": 9.813333333333332e-07,
+      "loss": 0.0002,
+      "reward": 1.375,
+      "reward_std": 0.1157275140285492,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 140
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 85.71875,
+      "epoch": 0.2256,
+      "grad_norm": 6.037929340064492,
+      "kl": 0.158203125,
+      "learning_rate": 9.811999999999998e-07,
+      "loss": 0.0002,
+      "reward": 1.546875,
+      "reward_std": 0.10205793008208275,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.546875,
+      "step": 141
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 87.78125,
+      "epoch": 0.2272,
+      "grad_norm": 5.840158910337413,
+      "kl": 0.1552734375,
+      "learning_rate": 9.810666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.53125,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.53125,
+      "step": 142
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 83.78125,
+      "epoch": 0.2288,
+      "grad_norm": 9.944768204184994,
+      "kl": 0.15771484375,
+      "learning_rate": 9.809333333333332e-07,
+      "loss": 0.0002,
+      "reward": 1.84375,
+      "reward_std": 0.0883883461356163,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.84375,
+      "step": 143
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 81.1875,
+      "epoch": 0.2304,
+      "grad_norm": 4.047283406148725,
+      "kl": 0.17236328125,
+      "learning_rate": 9.808e-07,
+      "loss": 0.0002,
+      "reward": 1.578125,
+      "reward_std": 0.12255740165710449,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.578125,
+      "step": 144
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 83.328125,
+      "epoch": 0.232,
+      "grad_norm": 4.420845219930897,
+      "kl": 0.1796875,
+      "learning_rate": 9.806666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.765625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.765625,
+      "step": 145
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 87.6875,
+      "epoch": 0.2336,
+      "grad_norm": 12.440323986319731,
+      "kl": 0.31494140625,
+      "learning_rate": 9.805333333333334e-07,
+      "loss": 0.0003,
+      "reward": 1.546875,
+      "reward_std": 0.25726158916950226,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.546875,
+      "step": 146
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 89.578125,
+      "epoch": 0.2352,
+      "grad_norm": 5.890106439558876,
+      "kl": 0.1494140625,
+      "learning_rate": 9.804e-07,
+      "loss": 0.0001,
+      "reward": 1.84375,
+      "reward_std": 0.0883883461356163,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.84375,
+      "step": 147
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 87.421875,
+      "epoch": 0.2368,
+      "grad_norm": 4.636241824214292,
+      "kl": 0.18017578125,
+      "learning_rate": 9.802666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.17570312321186066,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 148
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 89.671875,
+      "epoch": 0.2384,
+      "grad_norm": 2.868476167212098,
+      "kl": 0.15283203125,
+      "learning_rate": 9.801333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.765625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.765625,
+      "step": 149
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 88.53125,
+      "epoch": 0.24,
+      "grad_norm": 5.451437950319999,
+      "kl": 0.1611328125,
+      "learning_rate": 9.8e-07,
+      "loss": 0.0002,
+      "reward": 1.71875,
+      "reward_std": 0.0883883461356163,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.71875,
+      "step": 150
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 91.265625,
+      "epoch": 0.2416,
+      "grad_norm": 5.83491019506141,
+      "kl": 0.15771484375,
+      "learning_rate": 9.798666666666665e-07,
+      "loss": 0.0002,
+      "reward": 1.53125,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.53125,
+      "step": 151
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 87.9375,
+      "epoch": 0.2432,
+      "grad_norm": 29.649688821362563,
+      "kl": 0.158203125,
+      "learning_rate": 9.797333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 152
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 85.5,
+      "epoch": 0.2448,
+      "grad_norm": 13.427507576427232,
+      "kl": 0.16357421875,
+      "learning_rate": 9.796e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.2130674123764038,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.640625,
+      "step": 153
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 86.828125,
+      "epoch": 0.2464,
+      "grad_norm": 0.055841722372992925,
+      "kl": 0.17333984375,
+      "learning_rate": 9.794666666666667e-07,
+      "loss": 0.0002,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 154
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 89.453125,
+      "epoch": 0.248,
+      "grad_norm": 6.015825734642309,
+      "kl": 0.15234375,
+      "learning_rate": 9.793333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.46875,
+      "reward_std": 0.0883883461356163,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.46875,
+      "step": 155
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 88.5625,
+      "epoch": 0.2496,
+      "grad_norm": 4.928987026183256,
+      "kl": 0.158203125,
+      "learning_rate": 9.791999999999999e-07,
+      "loss": 0.0002,
+      "reward": 1.6875,
+      "reward_std": 0.10888782143592834,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.6875,
+      "step": 156
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 87.5,
+      "epoch": 0.2512,
+      "grad_norm": 0.031875363177478966,
+      "kl": 0.150390625,
+      "learning_rate": 9.790666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 157
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 89.265625,
+      "epoch": 0.2528,
+      "grad_norm": 0.017922801968365563,
+      "kl": 0.16552734375,
+      "learning_rate": 9.789333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 158
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 89.390625,
+      "epoch": 0.2544,
+      "grad_norm": 14.207425736657866,
+      "kl": 0.1728515625,
+      "learning_rate": 9.788e-07,
+      "loss": 0.0002,
+      "reward": 1.78125,
+      "reward_std": 0.19149437546730042,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.78125,
+      "step": 159
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 90.59375,
+      "epoch": 0.256,
+      "grad_norm": 11.170033553984178,
+      "kl": 0.16845703125,
+      "learning_rate": 9.786666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.515625,
+      "reward_std": 0.12255740165710449,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.515625,
+      "step": 160
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 94.78125,
+      "epoch": 0.2576,
+      "grad_norm": 7.42505829072604,
+      "kl": 0.16357421875,
+      "learning_rate": 9.785333333333332e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0883883461356163,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 161
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 94.171875,
+      "epoch": 0.2592,
+      "grad_norm": 7.331971839266302,
+      "kl": 0.18310546875,
+      "learning_rate": 9.784e-07,
+      "loss": 0.0002,
+      "reward": 1.640625,
+      "reward_std": 0.12255740165710449,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.640625,
+      "step": 162
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 95.078125,
+      "epoch": 0.2608,
+      "grad_norm": 5.4615360002040125,
+      "kl": 0.1689453125,
+      "learning_rate": 9.782666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.546875,
+      "reward_std": 0.10205793380737305,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.546875,
+      "step": 163
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 93.84375,
+      "epoch": 0.2624,
+      "grad_norm": 6.520576505381738,
+      "kl": 0.16796875,
+      "learning_rate": 9.781333333333332e-07,
+      "loss": 0.0002,
+      "reward": 1.46875,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.46875,
+      "step": 164
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 92.53125,
+      "epoch": 0.264,
+      "grad_norm": 3.854590689226584,
+      "kl": 0.16845703125,
+      "learning_rate": 9.78e-07,
+      "loss": 0.0002,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 165
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 90.609375,
+      "epoch": 0.2656,
+      "grad_norm": 6.408461749105565,
+      "kl": 0.17822265625,
+      "learning_rate": 9.778666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.53125,
+      "reward_std": 0.0883883461356163,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.53125,
+      "step": 166
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 91.671875,
+      "epoch": 0.2672,
+      "grad_norm": 4.789850563829869,
+      "kl": 0.21875,
+      "learning_rate": 9.777333333333334e-07,
+      "loss": 0.0002,
+      "reward": 1.5625,
+      "reward_std": 0.10888781771063805,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5625,
+      "step": 167
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 91.796875,
+      "epoch": 0.2688,
+      "grad_norm": 21.04329190706873,
+      "kl": 0.193359375,
+      "learning_rate": 9.776e-07,
+      "loss": 0.0002,
+      "reward": 1.4375,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.4375,
+      "step": 168
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 95.78125,
+      "epoch": 0.2704,
+      "grad_norm": 42.123553380020525,
+      "kl": 0.15966796875,
+      "learning_rate": 9.774666666666668e-07,
+      "loss": 0.0002,
+      "reward": 1.921875,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.921875,
+      "step": 169
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 94.625,
+      "epoch": 0.272,
+      "grad_norm": 9.594943593465862,
+      "kl": 0.1552734375,
+      "learning_rate": 9.773333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.5625,
+      "reward_std": 0.1157275140285492,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5625,
+      "step": 170
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 95.09375,
+      "epoch": 0.2736,
+      "grad_norm": 7.747735415116444,
+      "kl": 0.1650390625,
+      "learning_rate": 9.772e-07,
+      "loss": 0.0002,
+      "reward": 1.640625,
+      "reward_std": 0.10205793380737305,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.65625,
+      "step": 171
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 93.46875,
+      "epoch": 0.2752,
+      "grad_norm": 7.241458552829622,
+      "kl": 0.16455078125,
+      "learning_rate": 9.770666666666665e-07,
+      "loss": 0.0002,
+      "reward": 1.65625,
+      "reward_std": 0.10888781771063805,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.65625,
+      "step": 172
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 99.84375,
+      "epoch": 0.2768,
+      "grad_norm": 3.070537030094569,
+      "kl": 0.177734375,
+      "learning_rate": 9.769333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.703125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.703125,
+      "step": 173
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 100.09375,
+      "epoch": 0.2784,
+      "grad_norm": 4.870095039313274,
+      "kl": 0.17041015625,
+      "learning_rate": 9.768e-07,
+      "loss": 0.0002,
+      "reward": 1.59375,
+      "reward_std": 0.0883883461356163,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.59375,
+      "step": 174
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 95.328125,
+      "epoch": 0.28,
+      "grad_norm": 14.326584361280565,
+      "kl": 0.1845703125,
+      "learning_rate": 9.766666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.703125,
+      "reward_std": 0.16887324303388596,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.703125,
+      "step": 175
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 97.953125,
+      "epoch": 0.2816,
+      "grad_norm": 0.029087757880566383,
+      "kl": 0.1591796875,
+      "learning_rate": 9.765333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 176
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 97.65625,
+      "epoch": 0.2832,
+      "grad_norm": 0.02757333419364728,
+      "kl": 0.18212890625,
+      "learning_rate": 9.764e-07,
+      "loss": 0.0002,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 177
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 97.75,
+      "epoch": 0.2848,
+      "grad_norm": 6.486818026408196,
+      "kl": 0.15673828125,
+      "learning_rate": 9.762666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.484375,
+      "reward_std": 0.12255740165710449,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.484375,
+      "step": 178
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 96.15625,
+      "epoch": 0.2864,
+      "grad_norm": 0.0365184049450036,
+      "kl": 0.16162109375,
+      "learning_rate": 9.761333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 179
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 94.09375,
+      "epoch": 0.288,
+      "grad_norm": 3.0720989129276766,
+      "kl": 0.15625,
+      "learning_rate": 9.759999999999998e-07,
+      "loss": 0.0002,
+      "reward": 1.578125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.578125,
+      "step": 180
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 92.453125,
+      "epoch": 0.2896,
+      "grad_norm": 4.625604377349422,
+      "kl": 0.14306640625,
+      "learning_rate": 9.758666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.78125,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.78125,
+      "step": 181
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 99.421875,
+      "epoch": 0.2912,
+      "grad_norm": 19.628970155058592,
+      "kl": 0.16845703125,
+      "learning_rate": 9.757333333333332e-07,
+      "loss": 0.0002,
+      "reward": 1.59375,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.59375,
+      "step": 182
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 97.984375,
+      "epoch": 0.2928,
+      "grad_norm": 10.995257561857485,
+      "kl": 0.15673828125,
+      "learning_rate": 9.756e-07,
+      "loss": 0.0002,
+      "reward": 1.8125,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.8125,
+      "step": 183
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 99.65625,
+      "epoch": 0.2944,
+      "grad_norm": 0.014341193994917761,
+      "kl": 0.16064453125,
+      "learning_rate": 9.754666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 184
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 96.890625,
+      "epoch": 0.296,
+      "grad_norm": 6.475590303016227,
+      "kl": 0.14501953125,
+      "learning_rate": 9.753333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.40625,
+      "reward_std": 0.0883883461356163,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.40625,
+      "step": 185
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 100.46875,
+      "epoch": 0.2976,
+      "grad_norm": 0.02342736532466333,
+      "kl": 0.1689453125,
+      "learning_rate": 9.752e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 186
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 93.375,
+      "epoch": 0.2992,
+      "grad_norm": 0.032579724133919595,
+      "kl": 0.15185546875,
+      "learning_rate": 9.750666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 187
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 101.796875,
+      "epoch": 0.3008,
+      "grad_norm": 3.062840173846955,
+      "kl": 0.15966796875,
+      "learning_rate": 9.749333333333332e-07,
+      "loss": 0.0002,
+      "reward": 1.859375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.859375,
+      "step": 188
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 100.234375,
+      "epoch": 0.3024,
+      "grad_norm": 0.017634290581152477,
+      "kl": 0.16845703125,
+      "learning_rate": 9.748e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 189
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 100.1875,
+      "epoch": 0.304,
+      "grad_norm": 8.957147177409487,
+      "kl": 0.1650390625,
+      "learning_rate": 9.746666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.609375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.609375,
+      "step": 190
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 92.640625,
+      "epoch": 0.3056,
+      "grad_norm": 104.77381530056844,
+      "kl": 0.1474609375,
+      "learning_rate": 9.745333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.71875,
+      "reward_std": 0.17570313066244125,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.71875,
+      "step": 191
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 91.84375,
+      "epoch": 0.3072,
+      "grad_norm": 0.018633288356094343,
+      "kl": 0.1572265625,
+      "learning_rate": 9.744e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 192
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 97.90625,
+      "epoch": 0.3088,
+      "grad_norm": 3.5983340671603328,
+      "kl": 0.1689453125,
+      "learning_rate": 9.742666666666665e-07,
+      "loss": 0.0002,
+      "reward": 1.84375,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.84375,
+      "step": 193
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 95.609375,
+      "epoch": 0.3104,
+      "grad_norm": 13.561295740483141,
+      "kl": 0.1630859375,
+      "learning_rate": 9.741333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.765625,
+      "reward_std": 0.1804211586713791,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.765625,
+      "step": 194
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 93.75,
+      "epoch": 0.312,
+      "grad_norm": 5.333208447801494,
+      "kl": 0.17236328125,
+      "learning_rate": 9.74e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0883883461356163,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 195
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 97.453125,
+      "epoch": 0.3136,
+      "grad_norm": 7.111932979462083,
+      "kl": 0.150390625,
+      "learning_rate": 9.738666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.46875,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.46875,
+      "step": 196
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 94.703125,
+      "epoch": 0.3152,
+      "grad_norm": 0.017421799554187087,
+      "kl": 0.15869140625,
+      "learning_rate": 9.737333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 197
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 93.734375,
+      "epoch": 0.3168,
+      "grad_norm": 0.013743878238269967,
+      "kl": 0.15380859375,
+      "learning_rate": 9.735999999999999e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 198
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 99.75,
+      "epoch": 0.3184,
+      "grad_norm": 8.453884494998222,
+      "kl": 0.1669921875,
+      "learning_rate": 9.734666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.40625,
+      "reward_std": 0.1246790662407875,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.40625,
+      "step": 199
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 97.828125,
+      "epoch": 0.32,
+      "grad_norm": 6.901650050534742,
+      "kl": 0.16943359375,
+      "learning_rate": 9.733333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0883883461356163,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 200
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 100.21875,
+      "epoch": 0.3216,
+      "grad_norm": 6.459177267338049,
+      "kl": 0.1884765625,
+      "learning_rate": 9.731999999999998e-07,
+      "loss": 0.0002,
+      "reward": 1.71875,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.71875,
+      "step": 201
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 99.46875,
+      "epoch": 0.3232,
+      "grad_norm": 11.237915369584652,
+      "kl": 0.16748046875,
+      "learning_rate": 9.730666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.390625,
+      "reward_std": 0.10205793008208275,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.390625,
+      "step": 202
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 98.90625,
+      "epoch": 0.3248,
+      "grad_norm": 18.577719918744616,
+      "kl": 0.17236328125,
+      "learning_rate": 9.729333333333332e-07,
+      "loss": 0.0002,
+      "reward": 1.5625,
+      "reward_std": 0.16675157845020294,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5625,
+      "step": 203
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 92.4375,
+      "epoch": 0.3264,
+      "grad_norm": 1.2668513347483483,
+      "kl": 0.23193359375,
+      "learning_rate": 9.728e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 204
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 93.453125,
+      "epoch": 0.328,
+      "grad_norm": 0.022057264763760735,
+      "kl": 0.18505859375,
+      "learning_rate": 9.726666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 205
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 94.71875,
+      "epoch": 0.3296,
+      "grad_norm": 0.009904347788344006,
+      "kl": 0.15283203125,
+      "learning_rate": 9.725333333333334e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 206
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 103.8125,
+      "epoch": 0.3312,
+      "grad_norm": 0.016084025802647214,
+      "kl": 0.15185546875,
+      "learning_rate": 9.724e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 207
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 98.640625,
+      "epoch": 0.3328,
+      "grad_norm": 4.0504300153899475,
+      "kl": 0.16064453125,
+      "learning_rate": 9.722666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.796875,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.796875,
+      "step": 208
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 97.015625,
+      "epoch": 0.3344,
+      "grad_norm": 19.047222149514337,
+      "kl": 0.1708984375,
+      "learning_rate": 9.721333333333332e-07,
+      "loss": 0.0002,
+      "reward": 1.515625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.515625,
+      "step": 209
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.625,
+      "epoch": 0.336,
+      "grad_norm": 7.59549269681024,
+      "kl": 0.17578125,
+      "learning_rate": 9.72e-07,
+      "loss": 0.0002,
+      "reward": 1.5625,
+      "reward_std": 0.10888782143592834,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5625,
+      "step": 210
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 102.171875,
+      "epoch": 0.3376,
+      "grad_norm": 43.241493001752744,
+      "kl": 0.173828125,
+      "learning_rate": 9.718666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.296875,
+      "reward_std": 0.11100947856903076,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.296875,
+      "step": 211
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 98.34375,
+      "epoch": 0.3392,
+      "grad_norm": 6.744032183139456,
+      "kl": 0.1572265625,
+      "learning_rate": 9.717333333333334e-07,
+      "loss": 0.0002,
+      "reward": 1.46875,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.46875,
+      "step": 212
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.078125,
+      "epoch": 0.3408,
+      "grad_norm": 0.03814513581668419,
+      "kl": 0.15966796875,
+      "learning_rate": 9.716e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 213
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 102.46875,
+      "epoch": 0.3424,
+      "grad_norm": 6.909355411145973,
+      "kl": 0.1787109375,
+      "learning_rate": 9.714666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.578125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.578125,
+      "step": 214
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 103.3125,
+      "epoch": 0.344,
+      "grad_norm": 16.23752838506403,
+      "kl": 0.18115234375,
+      "learning_rate": 9.713333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0883883461356163,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 215
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 98.375,
+      "epoch": 0.3456,
+      "grad_norm": 0.029368766779067152,
+      "kl": 0.15576171875,
+      "learning_rate": 9.712e-07,
+      "loss": 0.0002,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 216
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 102.46875,
+      "epoch": 0.3472,
+      "grad_norm": 11.2745440199873,
+      "kl": 0.15673828125,
+      "learning_rate": 9.710666666666665e-07,
+      "loss": 0.0002,
+      "reward": 1.359375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.359375,
+      "step": 217
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.9375,
+      "epoch": 0.3488,
+      "grad_norm": 0.03898754948976148,
+      "kl": 0.1513671875,
+      "learning_rate": 9.709333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 218
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 103.703125,
+      "epoch": 0.3504,
+      "grad_norm": 6.223864003695034,
+      "kl": 0.15771484375,
+      "learning_rate": 9.707999999999999e-07,
+      "loss": 0.0002,
+      "reward": 1.578125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.578125,
+      "step": 219
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 103.21875,
+      "epoch": 0.352,
+      "grad_norm": 3.157833308866837,
+      "kl": 0.14599609375,
+      "learning_rate": 9.706666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.546875,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.546875,
+      "step": 220
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 101.609375,
+      "epoch": 0.3536,
+      "grad_norm": 8.253497914710932,
+      "kl": 0.14453125,
+      "learning_rate": 9.705333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.1875,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.1875,
+      "step": 221
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.34375,
+      "epoch": 0.3552,
+      "grad_norm": 49.72600360837356,
+      "kl": 0.146484375,
+      "learning_rate": 9.704e-07,
+      "loss": 0.0001,
+      "reward": 1.46875,
+      "reward_std": 0.17570312321186066,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.484375,
+      "step": 222
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 103.140625,
+      "epoch": 0.3568,
+      "grad_norm": 0.16566001590482266,
+      "kl": 0.16943359375,
+      "learning_rate": 9.702666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 223
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.5,
+      "epoch": 0.3584,
+      "grad_norm": 4.431726541634793,
+      "kl": 0.15283203125,
+      "learning_rate": 9.701333333333332e-07,
+      "loss": 0.0002,
+      "reward": 1.390625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.390625,
+      "step": 224
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.9375,
+      "epoch": 0.36,
+      "grad_norm": 3.870144210075645,
+      "kl": 0.15234375,
+      "learning_rate": 9.7e-07,
+      "loss": 0.0002,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 225
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 103.9375,
+      "epoch": 0.3616,
+      "grad_norm": 7.495517529398244,
+      "kl": 0.16357421875,
+      "learning_rate": 9.698666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.796875,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.796875,
+      "step": 226
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.234375,
+      "epoch": 0.3632,
+      "grad_norm": 0.011514032610728116,
+      "kl": 0.1435546875,
+      "learning_rate": 9.697333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 227
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 102.84375,
+      "epoch": 0.3648,
+      "grad_norm": 0.012377569869550636,
+      "kl": 0.15185546875,
+      "learning_rate": 9.696e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 228
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.296875,
+      "epoch": 0.3664,
+      "grad_norm": 0.03376406716667753,
+      "kl": 0.15234375,
+      "learning_rate": 9.694666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 229
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.96875,
+      "epoch": 0.368,
+      "grad_norm": 2.6611624000122127,
+      "kl": 0.1748046875,
+      "learning_rate": 9.693333333333334e-07,
+      "loss": 0.0002,
+      "reward": 1.46875,
+      "reward_std": 0.0883883461356163,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.484375,
+      "step": 230
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 103.765625,
+      "epoch": 0.3696,
+      "grad_norm": 0.2778041620911128,
+      "kl": 0.1689453125,
+      "learning_rate": 9.692e-07,
+      "loss": 0.0002,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 231
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.5,
+      "epoch": 0.3712,
+      "grad_norm": 0.03181594169820374,
+      "kl": 0.18310546875,
+      "learning_rate": 9.690666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 232
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.046875,
+      "epoch": 0.3728,
+      "grad_norm": 0.0311415005038891,
+      "kl": 0.16796875,
+      "learning_rate": 9.689333333333334e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 233
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.78125,
+      "epoch": 0.3744,
+      "grad_norm": 4.319270181581536,
+      "kl": 0.1591796875,
+      "learning_rate": 9.688e-07,
+      "loss": 0.0002,
+      "reward": 1.484375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.484375,
+      "step": 234
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.890625,
+      "epoch": 0.376,
+      "grad_norm": 3.153989272891152,
+      "kl": 0.15234375,
+      "learning_rate": 9.686666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.453125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.453125,
+      "step": 235
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.875,
+      "epoch": 0.3776,
+      "grad_norm": 0.021540671879433027,
+      "kl": 0.1708984375,
+      "learning_rate": 9.685333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 236
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.0,
+      "epoch": 0.3792,
+      "grad_norm": 0.011467553788645972,
+      "kl": 0.1640625,
+      "learning_rate": 9.684e-07,
+      "loss": 0.0002,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 237
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 100.0,
+      "epoch": 0.3808,
+      "grad_norm": 8.288178328711219,
+      "kl": 0.1494140625,
+      "learning_rate": 9.682666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.3125,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.3125,
+      "step": 238
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 101.4375,
+      "epoch": 0.3824,
+      "grad_norm": 0.0953605329595907,
+      "kl": 0.17822265625,
+      "learning_rate": 9.681333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 239
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.328125,
+      "epoch": 0.384,
+      "grad_norm": 6.04130242518147,
+      "kl": 0.1611328125,
+      "learning_rate": 9.679999999999999e-07,
+      "loss": 0.0002,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 240
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.140625,
+      "epoch": 0.3856,
+      "grad_norm": 0.022102012701449784,
+      "kl": 0.17041015625,
+      "learning_rate": 9.678666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 241
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 104.03125,
+      "epoch": 0.3872,
+      "grad_norm": 7.286297937319763,
+      "kl": 0.166015625,
+      "learning_rate": 9.677333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 242
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.640625,
+      "epoch": 0.3888,
+      "grad_norm": 0.03749282588974005,
+      "kl": 0.1640625,
+      "learning_rate": 9.676e-07,
+      "loss": 0.0002,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 243
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.28125,
+      "epoch": 0.3904,
+      "grad_norm": 0.03372189228017584,
+      "kl": 0.177734375,
+      "learning_rate": 9.674666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 244
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.625,
+      "epoch": 0.392,
+      "grad_norm": 0.03512822549628239,
+      "kl": 0.15283203125,
+      "learning_rate": 9.673333333333332e-07,
+      "loss": 0.0002,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 245
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.546875,
+      "epoch": 0.3936,
+      "grad_norm": 0.015138995941978112,
+      "kl": 0.1640625,
+      "learning_rate": 9.671999999999998e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 246
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.390625,
+      "epoch": 0.3952,
+      "grad_norm": 0.012861723350795665,
+      "kl": 0.1591796875,
+      "learning_rate": 9.670666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 247
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 104.203125,
+      "epoch": 0.3968,
+      "grad_norm": 0.03239246430602263,
+      "kl": 0.158203125,
+      "learning_rate": 9.669333333333332e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 248
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.921875,
+      "epoch": 0.3984,
+      "grad_norm": 0.024540768802089778,
+      "kl": 0.18994140625,
+      "learning_rate": 9.668e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 249
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.0625,
+      "epoch": 0.4,
+      "grad_norm": 0.01388552074073628,
+      "kl": 0.1630859375,
+      "learning_rate": 9.666666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 250
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.71875,
+      "epoch": 0.4016,
+      "grad_norm": 0.058334723405291986,
+      "kl": 0.16357421875,
+      "learning_rate": 9.665333333333334e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 251
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.046875,
+      "epoch": 0.4032,
+      "grad_norm": 112.44010928454462,
+      "kl": 0.15869140625,
+      "learning_rate": 9.664e-07,
+      "loss": 0.0002,
+      "reward": 1.578125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.578125,
+      "step": 252
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.609375,
+      "epoch": 0.4048,
+      "grad_norm": 1.6522600118158886,
+      "kl": 0.18603515625,
+      "learning_rate": 9.662666666666668e-07,
+      "loss": 0.0002,
+      "reward": 1.609375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.625,
+      "step": 253
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.84375,
+      "epoch": 0.4064,
+      "grad_norm": 0.02652141308854601,
+      "kl": 0.16796875,
+      "learning_rate": 9.661333333333331e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 254
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.421875,
+      "epoch": 0.408,
+      "grad_norm": 0.026896342265161424,
+      "kl": 0.169921875,
+      "learning_rate": 9.66e-07,
+      "loss": 0.0002,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 255
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.125,
+      "epoch": 0.4096,
+      "grad_norm": 0.041447616726995394,
+      "kl": 0.1806640625,
+      "learning_rate": 9.658666666666665e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 256
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.296875,
+      "epoch": 0.4112,
+      "grad_norm": 3.292933005371633,
+      "kl": 0.16162109375,
+      "learning_rate": 9.657333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.390625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.390625,
+      "step": 257
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.15625,
+      "epoch": 0.4128,
+      "grad_norm": 6.640413548885557,
+      "kl": 0.16552734375,
+      "learning_rate": 9.656e-07,
+      "loss": 0.0002,
+      "reward": 1.578125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.578125,
+      "step": 258
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.90625,
+      "epoch": 0.4144,
+      "grad_norm": 0.01814866317953886,
+      "kl": 0.15087890625,
+      "learning_rate": 9.654666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 259
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.90625,
+      "epoch": 0.416,
+      "grad_norm": 0.044786854637889326,
+      "kl": 0.14697265625,
+      "learning_rate": 9.653333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 260
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 104.546875,
+      "epoch": 0.4176,
+      "grad_norm": 0.021143952478628908,
+      "kl": 0.15234375,
+      "learning_rate": 9.651999999999999e-07,
+      "loss": 0.0002,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 261
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 104.46875,
+      "epoch": 0.4192,
+      "grad_norm": 3.1713346622725425,
+      "kl": 0.16259765625,
+      "learning_rate": 9.650666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.609375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.609375,
+      "step": 262
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.984375,
+      "epoch": 0.4208,
+      "grad_norm": 0.01994835952050489,
+      "kl": 0.1494140625,
+      "learning_rate": 9.649333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 263
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.4375,
+      "epoch": 0.4224,
+      "grad_norm": 9.318825905485506,
+      "kl": 0.1533203125,
+      "learning_rate": 9.647999999999999e-07,
+      "loss": 0.0002,
+      "reward": 1.4375,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.4375,
+      "step": 264
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.5625,
+      "epoch": 0.424,
+      "grad_norm": 0.017017383862292013,
+      "kl": 0.1552734375,
+      "learning_rate": 9.646666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.25,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.25,
+      "step": 265
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.5625,
+      "epoch": 0.4256,
+      "grad_norm": 0.1550472906102207,
+      "kl": 0.15673828125,
+      "learning_rate": 9.645333333333332e-07,
+      "loss": 0.0002,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 266
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.671875,
+      "epoch": 0.4272,
+      "grad_norm": 0.012861884562457454,
+      "kl": 0.1337890625,
+      "learning_rate": 9.644e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 267
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.84375,
+      "epoch": 0.4288,
+      "grad_norm": 0.10029229444772964,
+      "kl": 0.14453125,
+      "learning_rate": 9.642666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.125,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.125,
+      "step": 268
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.859375,
+      "epoch": 0.4304,
+      "grad_norm": 6.867707914479197,
+      "kl": 0.14306640625,
+      "learning_rate": 9.641333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.703125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.703125,
+      "step": 269
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.765625,
+      "epoch": 0.432,
+      "grad_norm": 13.460322300749821,
+      "kl": 0.1396484375,
+      "learning_rate": 9.64e-07,
+      "loss": 0.0001,
+      "reward": 1.390625,
+      "reward_std": 0.12255740165710449,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.390625,
+      "step": 270
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.15625,
+      "epoch": 0.4336,
+      "grad_norm": 6.742366488097734,
+      "kl": 0.138671875,
+      "learning_rate": 9.638666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.453125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.453125,
+      "step": 271
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.828125,
+      "epoch": 0.4352,
+      "grad_norm": 0.021536231478240464,
+      "kl": 0.13525390625,
+      "learning_rate": 9.637333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 272
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.6875,
+      "epoch": 0.4368,
+      "grad_norm": 0.02023831339570812,
+      "kl": 0.14013671875,
+      "learning_rate": 9.636e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 273
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.9375,
+      "epoch": 0.4384,
+      "grad_norm": 9.406784750155644,
+      "kl": 0.134765625,
+      "learning_rate": 9.634666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.8125,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.8125,
+      "step": 274
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.78125,
+      "epoch": 0.44,
+      "grad_norm": 7.296672688426473,
+      "kl": 0.12939453125,
+      "learning_rate": 9.633333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.859375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.859375,
+      "step": 275
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.328125,
+      "epoch": 0.4416,
+      "grad_norm": 0.013358703622745185,
+      "kl": 0.12646484375,
+      "learning_rate": 9.632e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 276
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.640625,
+      "epoch": 0.4432,
+      "grad_norm": 96.9658649125088,
+      "kl": 0.1376953125,
+      "learning_rate": 9.630666666666665e-07,
+      "loss": 0.0001,
+      "reward": 1.3125,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.3125,
+      "step": 277
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.078125,
+      "epoch": 0.4448,
+      "grad_norm": 25.21945003464881,
+      "kl": 0.138916015625,
+      "learning_rate": 9.629333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.671875,
+      "reward_std": 0.10205793008208275,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.671875,
+      "step": 278
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.859375,
+      "epoch": 0.4464,
+      "grad_norm": 3.5733033548055815,
+      "kl": 0.325927734375,
+      "learning_rate": 9.628e-07,
+      "loss": 0.0003,
+      "reward": 1.84375,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.84375,
+      "step": 279
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.6875,
+      "epoch": 0.448,
+      "grad_norm": 17.97738337458104,
+      "kl": 0.13623046875,
+      "learning_rate": 9.626666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.46875,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.46875,
+      "step": 280
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.578125,
+      "epoch": 0.4496,
+      "grad_norm": 0.0462841996032419,
+      "kl": 0.146484375,
+      "learning_rate": 9.625333333333333e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 281
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.90625,
+      "epoch": 0.4512,
+      "grad_norm": 0.057090106395162404,
+      "kl": 0.14794921875,
+      "learning_rate": 9.624e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 282
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.640625,
+      "epoch": 0.4528,
+      "grad_norm": 0.024413106846741153,
+      "kl": 0.1435546875,
+      "learning_rate": 9.622666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 283
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.015625,
+      "epoch": 0.4544,
+      "grad_norm": 0.024258118253456976,
+      "kl": 0.14501953125,
+      "learning_rate": 9.621333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 284
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.0625,
+      "epoch": 0.456,
+      "grad_norm": 0.03264572943696262,
+      "kl": 0.142578125,
+      "learning_rate": 9.619999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 285
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.8125,
+      "epoch": 0.4576,
+      "grad_norm": 7.282410145351634,
+      "kl": 0.15380859375,
+      "learning_rate": 9.618666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.671875,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.671875,
+      "step": 286
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.921875,
+      "epoch": 0.4592,
+      "grad_norm": 2.63395025979827,
+      "kl": 0.13818359375,
+      "learning_rate": 9.617333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.484375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.5,
+      "step": 287
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.546875,
+      "epoch": 0.4608,
+      "grad_norm": 0.033948498670509926,
+      "kl": 0.1376953125,
+      "learning_rate": 9.616e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 288
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.703125,
+      "epoch": 0.4624,
+      "grad_norm": 0.01687254051179704,
+      "kl": 0.1318359375,
+      "learning_rate": 9.614666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 289
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.15625,
+      "epoch": 0.464,
+      "grad_norm": 14.682353403629934,
+      "kl": 0.15380859375,
+      "learning_rate": 9.613333333333334e-07,
+      "loss": 0.0002,
+      "reward": 1.71875,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.71875,
+      "step": 290
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.21875,
+      "epoch": 0.4656,
+      "grad_norm": 49.94781926142795,
+      "kl": 0.14599609375,
+      "learning_rate": 9.612e-07,
+      "loss": 0.0001,
+      "reward": 1.59375,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.59375,
+      "step": 291
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.578125,
+      "epoch": 0.4672,
+      "grad_norm": 24.708072175957167,
+      "kl": 0.13720703125,
+      "learning_rate": 9.610666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.859375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.859375,
+      "step": 292
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.5625,
+      "epoch": 0.4688,
+      "grad_norm": 0.012967622130616457,
+      "kl": 0.130126953125,
+      "learning_rate": 9.609333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 293
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.203125,
+      "epoch": 0.4704,
+      "grad_norm": 0.023962211679779605,
+      "kl": 0.1279296875,
+      "learning_rate": 9.608e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 294
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.125,
+      "epoch": 0.472,
+      "grad_norm": 4.724952628959642,
+      "kl": 0.1455078125,
+      "learning_rate": 9.606666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.609375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.609375,
+      "step": 295
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.21875,
+      "epoch": 0.4736,
+      "grad_norm": 55.08954419304777,
+      "kl": 0.1298828125,
+      "learning_rate": 9.605333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.5625,
+      "reward_std": 0.23827511072158813,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5625,
+      "step": 296
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.140625,
+      "epoch": 0.4752,
+      "grad_norm": 0.023441500400181947,
+      "kl": 0.14013671875,
+      "learning_rate": 9.604e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 297
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.359375,
+      "epoch": 0.4768,
+      "grad_norm": 0.01582164220896833,
+      "kl": 0.146484375,
+      "learning_rate": 9.602666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 298
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.625,
+      "epoch": 0.4784,
+      "grad_norm": 28.25251839813154,
+      "kl": 0.1337890625,
+      "learning_rate": 9.601333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.59375,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.59375,
+      "step": 299
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.96875,
+      "epoch": 0.48,
+      "grad_norm": 0.019583182351057386,
+      "kl": 0.13720703125,
+      "learning_rate": 9.6e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 300
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.8125,
+      "epoch": 0.4816,
+      "grad_norm": 10.966017691703616,
+      "kl": 0.1435546875,
+      "learning_rate": 9.598666666666665e-07,
+      "loss": 0.0001,
+      "reward": 1.453125,
+      "reward_std": 0.12255740165710449,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.453125,
+      "step": 301
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.453125,
+      "epoch": 0.4832,
+      "grad_norm": 0.01999644817411647,
+      "kl": 0.123046875,
+      "learning_rate": 9.597333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 302
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.859375,
+      "epoch": 0.4848,
+      "grad_norm": 0.016530669607541643,
+      "kl": 0.119873046875,
+      "learning_rate": 9.595999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 303
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.515625,
+      "epoch": 0.4864,
+      "grad_norm": 0.012923994518905493,
+      "kl": 0.122314453125,
+      "learning_rate": 9.594666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 304
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.390625,
+      "epoch": 0.488,
+      "grad_norm": 0.03161664544344416,
+      "kl": 0.123779296875,
+      "learning_rate": 9.593333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 305
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.203125,
+      "epoch": 0.4896,
+      "grad_norm": 13.586541053840806,
+      "kl": 0.132080078125,
+      "learning_rate": 9.592e-07,
+      "loss": 0.0001,
+      "reward": 1.484375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.484375,
+      "step": 306
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.4375,
+      "epoch": 0.4912,
+      "grad_norm": 0.03475024329118867,
+      "kl": 0.134765625,
+      "learning_rate": 9.590666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 307
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.78125,
+      "epoch": 0.4928,
+      "grad_norm": 0.02000121984651751,
+      "kl": 0.134765625,
+      "learning_rate": 9.589333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 308
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.484375,
+      "epoch": 0.4944,
+      "grad_norm": 10.093175314339097,
+      "kl": 0.12841796875,
+      "learning_rate": 9.588e-07,
+      "loss": 0.0001,
+      "reward": 1.5625,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5625,
+      "step": 309
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.375,
+      "epoch": 0.496,
+      "grad_norm": 16.950484610210204,
+      "kl": 0.118896484375,
+      "learning_rate": 9.586666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.1157275140285492,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 310
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.734375,
+      "epoch": 0.4976,
+      "grad_norm": 70.51696142002126,
+      "kl": 0.134033203125,
+      "learning_rate": 9.585333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.703125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.703125,
+      "step": 311
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.296875,
+      "epoch": 0.4992,
+      "grad_norm": 21.64581858167852,
+      "kl": 0.130126953125,
+      "learning_rate": 9.584e-07,
+      "loss": 0.0001,
+      "reward": 1.484375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.484375,
+      "step": 312
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.234375,
+      "epoch": 0.5008,
+      "grad_norm": 0.025703063412914992,
+      "kl": 0.13330078125,
+      "learning_rate": 9.582666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 313
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.125,
+      "epoch": 0.5024,
+      "grad_norm": 23.93902605999519,
+      "kl": 0.13818359375,
+      "learning_rate": 9.581333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.71875,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.71875,
+      "step": 314
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.96875,
+      "epoch": 0.504,
+      "grad_norm": 7.157251978227721,
+      "kl": 0.11328125,
+      "learning_rate": 9.58e-07,
+      "loss": 0.0001,
+      "reward": 1.390625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.390625,
+      "step": 315
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.09375,
+      "epoch": 0.5056,
+      "grad_norm": 0.025033566000987526,
+      "kl": 0.1162109375,
+      "learning_rate": 9.578666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 316
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.046875,
+      "epoch": 0.5072,
+      "grad_norm": 0.009711970128793964,
+      "kl": 0.11865234375,
+      "learning_rate": 9.577333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 317
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.625,
+      "epoch": 0.5088,
+      "grad_norm": 0.012097295080929518,
+      "kl": 0.11865234375,
+      "learning_rate": 9.576e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 318
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.65625,
+      "epoch": 0.5104,
+      "grad_norm": 0.011370823902017447,
+      "kl": 0.12646484375,
+      "learning_rate": 9.574666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 319
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 141.640625,
+      "epoch": 0.512,
+      "grad_norm": 15.693303046479802,
+      "kl": 0.124267578125,
+      "learning_rate": 9.573333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.1157275140285492,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 320
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.046875,
+      "epoch": 0.5136,
+      "grad_norm": 16.18943019427335,
+      "kl": 0.1123046875,
+      "learning_rate": 9.572e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0883883461356163,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 321
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.15625,
+      "epoch": 0.5152,
+      "grad_norm": 0.014274977123746055,
+      "kl": 0.11474609375,
+      "learning_rate": 9.570666666666665e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 322
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.203125,
+      "epoch": 0.5168,
+      "grad_norm": 0.055829464275399636,
+      "kl": 0.1259765625,
+      "learning_rate": 9.569333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 323
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.625,
+      "epoch": 0.5184,
+      "grad_norm": 4.268200732805205,
+      "kl": 0.12109375,
+      "learning_rate": 9.567999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.84375,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.84375,
+      "step": 324
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.640625,
+      "epoch": 0.52,
+      "grad_norm": 0.02123370495375839,
+      "kl": 0.1123046875,
+      "learning_rate": 9.566666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 325
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.1875,
+      "epoch": 0.5216,
+      "grad_norm": 0.018750941886293986,
+      "kl": 0.098876953125,
+      "learning_rate": 9.565333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 326
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 144.15625,
+      "epoch": 0.5232,
+      "grad_norm": 0.00913571654230817,
+      "kl": 0.11328125,
+      "learning_rate": 9.564e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 327
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 142.9375,
+      "epoch": 0.5248,
+      "grad_norm": 0.02470257229203827,
+      "kl": 0.118896484375,
+      "learning_rate": 9.562666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 328
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 145.75,
+      "epoch": 0.5264,
+      "grad_norm": 0.015956749907938918,
+      "kl": 0.098876953125,
+      "learning_rate": 9.561333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 329
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 144.734375,
+      "epoch": 0.528,
+      "grad_norm": 0.011448759826872828,
+      "kl": 0.1005859375,
+      "learning_rate": 9.559999999999998e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 330
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.953125,
+      "epoch": 0.5296,
+      "grad_norm": 0.14715249398411956,
+      "kl": 0.11181640625,
+      "learning_rate": 9.558666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 331
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 153.3125,
+      "epoch": 0.5312,
+      "grad_norm": 6.295362918813196,
+      "kl": 0.09765625,
+      "learning_rate": 9.557333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.578125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.578125,
+      "step": 332
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 160.296875,
+      "epoch": 0.5328,
+      "grad_norm": 2.214708257085737,
+      "kl": 0.103271484375,
+      "learning_rate": 9.556e-07,
+      "loss": 0.0001,
+      "reward": 1.453125,
+      "reward_std": 0.13258251920342445,
+      "rewards/format_reward": 0.96875,
+      "rewards/score_reward": 0.484375,
+      "step": 333
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.703125,
+      "epoch": 0.5344,
+      "grad_norm": 0.009027825306941314,
+      "kl": 0.10302734375,
+      "learning_rate": 9.554666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 334
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.203125,
+      "epoch": 0.536,
+      "grad_norm": 0.016259900699034647,
+      "kl": 0.09326171875,
+      "learning_rate": 9.553333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 335
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.40625,
+      "epoch": 0.5376,
+      "grad_norm": 0.023345347451007078,
+      "kl": 0.10693359375,
+      "learning_rate": 9.552e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 336
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.40625,
+      "epoch": 0.5392,
+      "grad_norm": 0.04029713077301477,
+      "kl": 0.10009765625,
+      "learning_rate": 9.550666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 337
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.1875,
+      "epoch": 0.5408,
+      "grad_norm": 0.014882996784452752,
+      "kl": 0.099609375,
+      "learning_rate": 9.549333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 338
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.171875,
+      "epoch": 0.5424,
+      "grad_norm": 0.014202315138734732,
+      "kl": 0.090576171875,
+      "learning_rate": 9.548e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 339
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.5,
+      "epoch": 0.544,
+      "grad_norm": 0.01341271731286831,
+      "kl": 0.113525390625,
+      "learning_rate": 9.546666666666665e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 340
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.3125,
+      "epoch": 0.5456,
+      "grad_norm": 0.01415075022545253,
+      "kl": 0.103515625,
+      "learning_rate": 9.545333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 341
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.375,
+      "epoch": 0.5472,
+      "grad_norm": 0.012784723348356546,
+      "kl": 0.100341796875,
+      "learning_rate": 9.544e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 342
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.671875,
+      "epoch": 0.5488,
+      "grad_norm": 0.02337345449556848,
+      "kl": 0.106201171875,
+      "learning_rate": 9.542666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 343
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.0625,
+      "epoch": 0.5504,
+      "grad_norm": 4.286474802239919,
+      "kl": 0.0986328125,
+      "learning_rate": 9.541333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.84375,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.84375,
+      "step": 344
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.078125,
+      "epoch": 0.552,
+      "grad_norm": 0.020339707379178673,
+      "kl": 0.09912109375,
+      "learning_rate": 9.539999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 345
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.84375,
+      "epoch": 0.5536,
+      "grad_norm": 0.020437303648655494,
+      "kl": 0.09521484375,
+      "learning_rate": 9.538666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 346
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.703125,
+      "epoch": 0.5552,
+      "grad_norm": 0.02881903051397082,
+      "kl": 0.10205078125,
+      "learning_rate": 9.537333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 347
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.171875,
+      "epoch": 0.5568,
+      "grad_norm": 0.018372340256943155,
+      "kl": 0.106689453125,
+      "learning_rate": 9.536e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 348
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.328125,
+      "epoch": 0.5584,
+      "grad_norm": 0.04527299117186626,
+      "kl": 0.1044921875,
+      "learning_rate": 9.534666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 349
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.203125,
+      "epoch": 0.56,
+      "grad_norm": 11.297290854554157,
+      "kl": 0.1015625,
+      "learning_rate": 9.533333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 350
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.5616,
+      "grad_norm": 0.013354599505850938,
+      "kl": 0.1064453125,
+      "learning_rate": 9.532e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 351
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.484375,
+      "epoch": 0.5632,
+      "grad_norm": 0.01429338717199609,
+      "kl": 0.097900390625,
+      "learning_rate": 9.530666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 352
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.375,
+      "epoch": 0.5648,
+      "grad_norm": 0.009377051511256167,
+      "kl": 0.095947265625,
+      "learning_rate": 9.529333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 353
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.6875,
+      "epoch": 0.5664,
+      "grad_norm": 12.067769934792022,
+      "kl": 0.10400390625,
+      "learning_rate": 9.527999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.515625,
+      "reward_std": 0.13258251920342445,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.515625,
+      "step": 354
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.28125,
+      "epoch": 0.568,
+      "grad_norm": 15.810147954461765,
+      "kl": 0.097412109375,
+      "learning_rate": 9.526666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.671875,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.671875,
+      "step": 355
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.53125,
+      "epoch": 0.5696,
+      "grad_norm": 0.009938487828265955,
+      "kl": 0.102294921875,
+      "learning_rate": 9.525333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 356
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.859375,
+      "epoch": 0.5712,
+      "grad_norm": 0.012586120697414433,
+      "kl": 0.109375,
+      "learning_rate": 9.524e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 357
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.78125,
+      "epoch": 0.5728,
+      "grad_norm": 0.013445918752362392,
+      "kl": 0.111328125,
+      "learning_rate": 9.522666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 358
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.3125,
+      "epoch": 0.5744,
+      "grad_norm": 0.009647461647575068,
+      "kl": 0.1005859375,
+      "learning_rate": 9.521333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 359
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.015625,
+      "epoch": 0.576,
+      "grad_norm": 0.00904250451057301,
+      "kl": 0.1005859375,
+      "learning_rate": 9.52e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 360
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.6875,
+      "epoch": 0.5776,
+      "grad_norm": 0.057141902399688936,
+      "kl": 0.09716796875,
+      "learning_rate": 9.518666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 361
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.21875,
+      "epoch": 0.5792,
+      "grad_norm": 8.9375547361974,
+      "kl": 0.099609375,
+      "learning_rate": 9.517333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.71875,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.71875,
+      "step": 362
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.609375,
+      "epoch": 0.5808,
+      "grad_norm": 5.619514523513027,
+      "kl": 0.10546875,
+      "learning_rate": 9.515999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.78125,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.78125,
+      "step": 363
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.71875,
+      "epoch": 0.5824,
+      "grad_norm": 0.013100546153127224,
+      "kl": 0.121337890625,
+      "learning_rate": 9.514666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 364
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.828125,
+      "epoch": 0.584,
+      "grad_norm": 0.011320994133747053,
+      "kl": 0.10205078125,
+      "learning_rate": 9.513333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 365
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.453125,
+      "epoch": 0.5856,
+      "grad_norm": 11.950586588000629,
+      "kl": 0.099609375,
+      "learning_rate": 9.512e-07,
+      "loss": 0.0001,
+      "reward": 1.515625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.515625,
+      "step": 366
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.921875,
+      "epoch": 0.5872,
+      "grad_norm": 0.08458943636517062,
+      "kl": 0.11474609375,
+      "learning_rate": 9.510666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 367
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.234375,
+      "epoch": 0.5888,
+      "grad_norm": 0.016614050332397996,
+      "kl": 0.10888671875,
+      "learning_rate": 9.509333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 368
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.078125,
+      "epoch": 0.5904,
+      "grad_norm": 0.009679595069964273,
+      "kl": 0.0966796875,
+      "learning_rate": 9.508e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 369
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.4375,
+      "epoch": 0.592,
+      "grad_norm": 0.054261211055597315,
+      "kl": 0.095703125,
+      "learning_rate": 9.506666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 370
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.34375,
+      "epoch": 0.5936,
+      "grad_norm": 13.241023085009012,
+      "kl": 0.10009765625,
+      "learning_rate": 9.505333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.515625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.515625,
+      "step": 371
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.59375,
+      "epoch": 0.5952,
+      "grad_norm": 0.011872117744898525,
+      "kl": 0.10107421875,
+      "learning_rate": 9.503999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 372
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.1875,
+      "epoch": 0.5968,
+      "grad_norm": 0.009473089033458535,
+      "kl": 0.10791015625,
+      "learning_rate": 9.502666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 373
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.296875,
+      "epoch": 0.5984,
+      "grad_norm": 0.013729704450194544,
+      "kl": 0.102783203125,
+      "learning_rate": 9.501333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.125,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.125,
+      "step": 374
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.078125,
+      "epoch": 0.6,
+      "grad_norm": 0.01641071110870672,
+      "kl": 0.09912109375,
+      "learning_rate": 9.499999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 375
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.25,
+      "epoch": 0.6016,
+      "grad_norm": 0.01655499866538641,
+      "kl": 0.1064453125,
+      "learning_rate": 9.498666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 376
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.8125,
+      "epoch": 0.6032,
+      "grad_norm": 0.009848852026434131,
+      "kl": 0.108642578125,
+      "learning_rate": 9.497333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 377
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.40625,
+      "epoch": 0.6048,
+      "grad_norm": 0.012512638991998658,
+      "kl": 0.111572265625,
+      "learning_rate": 9.496e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 378
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.8125,
+      "epoch": 0.6064,
+      "grad_norm": 0.009367901288739329,
+      "kl": 0.1044921875,
+      "learning_rate": 9.494666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 379
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.171875,
+      "epoch": 0.608,
+      "grad_norm": 24.083401860591827,
+      "kl": 0.09130859375,
+      "learning_rate": 9.493333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.40625,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.40625,
+      "step": 380
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.15625,
+      "epoch": 0.6096,
+      "grad_norm": 0.01082077641219462,
+      "kl": 0.109375,
+      "learning_rate": 9.492e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 381
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.78125,
+      "epoch": 0.6112,
+      "grad_norm": 7.915145030773695,
+      "kl": 0.135498046875,
+      "learning_rate": 9.490666666666665e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0883883461356163,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 382
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.375,
+      "epoch": 0.6128,
+      "grad_norm": 0.011220613001804542,
+      "kl": 0.107177734375,
+      "learning_rate": 9.489333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 383
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.671875,
+      "epoch": 0.6144,
+      "grad_norm": 1.6327065254535542,
+      "kl": 0.11865234375,
+      "learning_rate": 9.487999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.484375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.484375,
+      "step": 384
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.59375,
+      "epoch": 0.616,
+      "grad_norm": 0.03477778389784228,
+      "kl": 0.101806640625,
+      "learning_rate": 9.486666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 385
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.859375,
+      "epoch": 0.6176,
+      "grad_norm": 0.015959690445574153,
+      "kl": 0.1083984375,
+      "learning_rate": 9.485333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 386
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.578125,
+      "epoch": 0.6192,
+      "grad_norm": 26.87860174513818,
+      "kl": 0.091552734375,
+      "learning_rate": 9.484e-07,
+      "loss": 0.0001,
+      "reward": 1.5625,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5625,
+      "step": 387
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.515625,
+      "epoch": 0.6208,
+      "grad_norm": 0.01229198053572687,
+      "kl": 0.102294921875,
+      "learning_rate": 9.482666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 388
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.953125,
+      "epoch": 0.6224,
+      "grad_norm": 0.010508717433475102,
+      "kl": 0.095458984375,
+      "learning_rate": 9.481333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 389
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.4375,
+      "epoch": 0.624,
+      "grad_norm": 0.009347105045429828,
+      "kl": 0.0966796875,
+      "learning_rate": 9.479999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 390
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.6875,
+      "epoch": 0.6256,
+      "grad_norm": 0.01510657216997804,
+      "kl": 0.097412109375,
+      "learning_rate": 9.478666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 391
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.828125,
+      "epoch": 0.6272,
+      "grad_norm": 26.307998341578664,
+      "kl": 0.09375,
+      "learning_rate": 9.477333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.640625,
+      "reward_std": 0.1315089538693428,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.640625,
+      "step": 392
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.890625,
+      "epoch": 0.6288,
+      "grad_norm": 17.948366289108392,
+      "kl": 0.097412109375,
+      "learning_rate": 9.475999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.65625,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.65625,
+      "step": 393
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.328125,
+      "epoch": 0.6304,
+      "grad_norm": 0.023810235742566566,
+      "kl": 0.109375,
+      "learning_rate": 9.474666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 394
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.3125,
+      "epoch": 0.632,
+      "grad_norm": 0.008118100439317169,
+      "kl": 0.09765625,
+      "learning_rate": 9.473333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 395
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.109375,
+      "epoch": 0.6336,
+      "grad_norm": 0.011308436102722303,
+      "kl": 0.1083984375,
+      "learning_rate": 9.472e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 396
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.25,
+      "epoch": 0.6352,
+      "grad_norm": 0.029790967559476727,
+      "kl": 0.104736328125,
+      "learning_rate": 9.470666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 397
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.234375,
+      "epoch": 0.6368,
+      "grad_norm": 0.012732299836509918,
+      "kl": 0.105224609375,
+      "learning_rate": 9.469333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 398
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.859375,
+      "epoch": 0.6384,
+      "grad_norm": 0.007388222417925291,
+      "kl": 0.10009765625,
+      "learning_rate": 9.468e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 399
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.515625,
+      "epoch": 0.64,
+      "grad_norm": 0.012522710133058475,
+      "kl": 0.105224609375,
+      "learning_rate": 9.466666666666666e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 400
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.21875,
+      "epoch": 0.6416,
+      "grad_norm": 0.009666852359804022,
+      "kl": 0.104736328125,
+      "learning_rate": 9.465333333333333e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 401
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.84375,
+      "epoch": 0.6432,
+      "grad_norm": 3.0539562207463122,
+      "kl": 0.107666015625,
+      "learning_rate": 9.464e-07,
+      "loss": 0.0001,
+      "reward": 1.484375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.484375,
+      "step": 402
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.375,
+      "epoch": 0.6448,
+      "grad_norm": 0.030386362327855576,
+      "kl": 0.09912109375,
+      "learning_rate": 9.462666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 403
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.046875,
+      "epoch": 0.6464,
+      "grad_norm": 0.014853440482562864,
+      "kl": 0.117919921875,
+      "learning_rate": 9.461333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 404
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.328125,
+      "epoch": 0.648,
+      "grad_norm": 0.01816808399373388,
+      "kl": 0.10205078125,
+      "learning_rate": 9.459999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 405
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.421875,
+      "epoch": 0.6496,
+      "grad_norm": 0.01483670016059827,
+      "kl": 0.10888671875,
+      "learning_rate": 9.458666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 406
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.5,
+      "epoch": 0.6512,
+      "grad_norm": 0.01248407737054074,
+      "kl": 0.102294921875,
+      "learning_rate": 9.457333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 407
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.90625,
+      "epoch": 0.6528,
+      "grad_norm": 0.009655020042683904,
+      "kl": 0.09716796875,
+      "learning_rate": 9.456e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 408
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.953125,
+      "epoch": 0.6544,
+      "grad_norm": 0.011406101571721136,
+      "kl": 0.116943359375,
+      "learning_rate": 9.454666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 409
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.015625,
+      "epoch": 0.656,
+      "grad_norm": 0.010409445558255504,
+      "kl": 0.107666015625,
+      "learning_rate": 9.453333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 410
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.765625,
+      "epoch": 0.6576,
+      "grad_norm": 13.543185265640394,
+      "kl": 0.099853515625,
+      "learning_rate": 9.452e-07,
+      "loss": 0.0001,
+      "reward": 1.6875,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.6875,
+      "step": 411
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.03125,
+      "epoch": 0.6592,
+      "grad_norm": 0.011387845545426234,
+      "kl": 0.1064453125,
+      "learning_rate": 9.450666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 412
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.09375,
+      "epoch": 0.6608,
+      "grad_norm": 6.587113590623218,
+      "kl": 0.105712890625,
+      "learning_rate": 9.449333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.796875,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.796875,
+      "step": 413
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.4375,
+      "epoch": 0.6624,
+      "grad_norm": 0.009908664875979854,
+      "kl": 0.100830078125,
+      "learning_rate": 9.447999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 414
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.703125,
+      "epoch": 0.664,
+      "grad_norm": 0.014583896030487107,
+      "kl": 0.1005859375,
+      "learning_rate": 9.446666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 415
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.828125,
+      "epoch": 0.6656,
+      "grad_norm": 0.01231764649940579,
+      "kl": 0.114990234375,
+      "learning_rate": 9.445333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 416
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.546875,
+      "epoch": 0.6672,
+      "grad_norm": 0.010106490304112606,
+      "kl": 0.100341796875,
+      "learning_rate": 9.444e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 417
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.796875,
+      "epoch": 0.6688,
+      "grad_norm": 0.027168505981368392,
+      "kl": 0.101806640625,
+      "learning_rate": 9.442666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 418
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.03125,
+      "epoch": 0.6704,
+      "grad_norm": 0.02258617800589035,
+      "kl": 0.10546875,
+      "learning_rate": 9.441333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 419
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.90625,
+      "epoch": 0.672,
+      "grad_norm": 8.262148748852834,
+      "kl": 0.09521484375,
+      "learning_rate": 9.439999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.15625,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.15625,
+      "step": 420
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.921875,
+      "epoch": 0.6736,
+      "grad_norm": 9.600846313625292,
+      "kl": 0.119384765625,
+      "learning_rate": 9.438666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.484375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.484375,
+      "step": 421
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.15625,
+      "epoch": 0.6752,
+      "grad_norm": 0.01597514502576837,
+      "kl": 0.099365234375,
+      "learning_rate": 9.437333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 422
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.453125,
+      "epoch": 0.6768,
+      "grad_norm": 8.679794089183815,
+      "kl": 0.1064453125,
+      "learning_rate": 9.436e-07,
+      "loss": 0.0001,
+      "reward": 1.453125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.453125,
+      "step": 423
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.8125,
+      "epoch": 0.6784,
+      "grad_norm": 0.011943436982725047,
+      "kl": 0.116455078125,
+      "learning_rate": 9.434666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 424
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.4375,
+      "epoch": 0.68,
+      "grad_norm": 12.435119965674762,
+      "kl": 0.10791015625,
+      "learning_rate": 9.433333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 425
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.265625,
+      "epoch": 0.6816,
+      "grad_norm": 27.888112505647026,
+      "kl": 0.110595703125,
+      "learning_rate": 9.432e-07,
+      "loss": 0.0001,
+      "reward": 1.609375,
+      "reward_std": 0.15992169082164764,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.609375,
+      "step": 426
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.015625,
+      "epoch": 0.6832,
+      "grad_norm": 0.007850283819843746,
+      "kl": 0.100830078125,
+      "learning_rate": 9.430666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 427
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.0625,
+      "epoch": 0.6848,
+      "grad_norm": 0.0516319004105797,
+      "kl": 0.104248046875,
+      "learning_rate": 9.429333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 428
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.453125,
+      "epoch": 0.6864,
+      "grad_norm": 0.01743235804495904,
+      "kl": 0.089599609375,
+      "learning_rate": 9.427999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 429
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.71875,
+      "epoch": 0.688,
+      "grad_norm": 11.371236221440682,
+      "kl": 0.09912109375,
+      "learning_rate": 9.426666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 430
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.03125,
+      "epoch": 0.6896,
+      "grad_norm": 11.72621032614539,
+      "kl": 0.095703125,
+      "learning_rate": 9.425333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.359375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.359375,
+      "step": 431
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.75,
+      "epoch": 0.6912,
+      "grad_norm": 0.009983267121849746,
+      "kl": 0.09130859375,
+      "learning_rate": 9.424e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 432
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.296875,
+      "epoch": 0.6928,
+      "grad_norm": 14.253548153240683,
+      "kl": 0.095458984375,
+      "learning_rate": 9.422666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.65625,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.65625,
+      "step": 433
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.296875,
+      "epoch": 0.6944,
+      "grad_norm": 0.023082105694260127,
+      "kl": 0.10595703125,
+      "learning_rate": 9.421333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 434
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.5,
+      "epoch": 0.696,
+      "grad_norm": 0.02006932229155699,
+      "kl": 0.097900390625,
+      "learning_rate": 9.419999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 435
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.09375,
+      "epoch": 0.6976,
+      "grad_norm": 9.047468694678493,
+      "kl": 0.088623046875,
+      "learning_rate": 9.418666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.8125,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.8125,
+      "step": 436
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.0,
+      "epoch": 0.6992,
+      "grad_norm": 0.01544328927513908,
+      "kl": 0.100341796875,
+      "learning_rate": 9.417333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 437
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.125,
+      "epoch": 0.7008,
+      "grad_norm": 0.009614156816473601,
+      "kl": 0.091796875,
+      "learning_rate": 9.415999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 438
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.390625,
+      "epoch": 0.7024,
+      "grad_norm": 0.00940820498068906,
+      "kl": 0.09912109375,
+      "learning_rate": 9.414666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 439
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.484375,
+      "epoch": 0.704,
+      "grad_norm": 0.009621675009765508,
+      "kl": 0.102783203125,
+      "learning_rate": 9.413333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 440
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.78125,
+      "epoch": 0.7056,
+      "grad_norm": 0.019042543051049087,
+      "kl": 0.102783203125,
+      "learning_rate": 9.412e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 441
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.5,
+      "epoch": 0.7072,
+      "grad_norm": 0.012419245103731285,
+      "kl": 0.104736328125,
+      "learning_rate": 9.410666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 442
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.015625,
+      "epoch": 0.7088,
+      "grad_norm": 0.015155784775657417,
+      "kl": 0.097412109375,
+      "learning_rate": 9.409333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 443
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.0,
+      "epoch": 0.7104,
+      "grad_norm": 0.014913206200594286,
+      "kl": 0.0908203125,
+      "learning_rate": 9.408e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 444
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.71875,
+      "epoch": 0.712,
+      "grad_norm": 6.087150740475082,
+      "kl": 0.09521484375,
+      "learning_rate": 9.406666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.390625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.390625,
+      "step": 445
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.109375,
+      "epoch": 0.7136,
+      "grad_norm": 0.00998342787483718,
+      "kl": 0.098876953125,
+      "learning_rate": 9.405333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 446
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.0625,
+      "epoch": 0.7152,
+      "grad_norm": 0.09737376681505364,
+      "kl": 0.091552734375,
+      "learning_rate": 9.403999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 447
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.125,
+      "epoch": 0.7168,
+      "grad_norm": 0.007926542515219981,
+      "kl": 0.089111328125,
+      "learning_rate": 9.402666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 448
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.609375,
+      "epoch": 0.7184,
+      "grad_norm": 0.012721612530559092,
+      "kl": 0.103759765625,
+      "learning_rate": 9.401333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 449
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.421875,
+      "epoch": 0.72,
+      "grad_norm": 0.011339902923734742,
+      "kl": 0.103271484375,
+      "learning_rate": 9.399999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 450
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.5,
+      "epoch": 0.7216,
+      "grad_norm": 0.016098741340360184,
+      "kl": 0.09814453125,
+      "learning_rate": 9.398666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 451
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.796875,
+      "epoch": 0.7232,
+      "grad_norm": 0.018835533899994768,
+      "kl": 0.094482421875,
+      "learning_rate": 9.397333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 452
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.40625,
+      "epoch": 0.7248,
+      "grad_norm": 0.014843772077515032,
+      "kl": 0.099609375,
+      "learning_rate": 9.396e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 453
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.453125,
+      "epoch": 0.7264,
+      "grad_norm": 0.008894952403564868,
+      "kl": 0.091552734375,
+      "learning_rate": 9.394666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 454
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.8125,
+      "epoch": 0.728,
+      "grad_norm": 6.07609743274511,
+      "kl": 0.08740234375,
+      "learning_rate": 9.393333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.84375,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.84375,
+      "step": 455
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.890625,
+      "epoch": 0.7296,
+      "grad_norm": 0.018805986410176084,
+      "kl": 0.092529296875,
+      "learning_rate": 9.391999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 456
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.5625,
+      "epoch": 0.7312,
+      "grad_norm": 8.958609661299693,
+      "kl": 0.099365234375,
+      "learning_rate": 9.390666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.765625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.765625,
+      "step": 457
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.75,
+      "epoch": 0.7328,
+      "grad_norm": 0.01399664482141374,
+      "kl": 0.097412109375,
+      "learning_rate": 9.389333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 458
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.15625,
+      "epoch": 0.7344,
+      "grad_norm": 0.11044854064844108,
+      "kl": 0.095703125,
+      "learning_rate": 9.387999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 459
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.703125,
+      "epoch": 0.736,
+      "grad_norm": 11.54696280359905,
+      "kl": 0.093505859375,
+      "learning_rate": 9.386666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.640625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.640625,
+      "step": 460
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.0625,
+      "epoch": 0.7376,
+      "grad_norm": 0.009934622374248036,
+      "kl": 0.093994140625,
+      "learning_rate": 9.385333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 461
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.359375,
+      "epoch": 0.7392,
+      "grad_norm": 0.016376865232333674,
+      "kl": 0.08837890625,
+      "learning_rate": 9.384e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 462
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.375,
+      "epoch": 0.7408,
+      "grad_norm": 0.00994071180477854,
+      "kl": 0.08154296875,
+      "learning_rate": 9.382666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 463
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.28125,
+      "epoch": 0.7424,
+      "grad_norm": 0.03039757918045616,
+      "kl": 0.090087890625,
+      "learning_rate": 9.381333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 464
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.484375,
+      "epoch": 0.744,
+      "grad_norm": 0.01618999336948946,
+      "kl": 0.095458984375,
+      "learning_rate": 9.379999999999998e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 465
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.890625,
+      "epoch": 0.7456,
+      "grad_norm": 40.65247312687983,
+      "kl": 0.0888671875,
+      "learning_rate": 9.378666666666665e-07,
+      "loss": 0.0001,
+      "reward": 1.28125,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.28125,
+      "step": 466
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.515625,
+      "epoch": 0.7472,
+      "grad_norm": 13.516577724350729,
+      "kl": 0.0888671875,
+      "learning_rate": 9.377333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.390625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.390625,
+      "step": 467
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.984375,
+      "epoch": 0.7488,
+      "grad_norm": 8.73155977889445,
+      "kl": 0.091552734375,
+      "learning_rate": 9.375999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.71875,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.71875,
+      "step": 468
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.5625,
+      "epoch": 0.7504,
+      "grad_norm": 0.008652542787636842,
+      "kl": 0.081298828125,
+      "learning_rate": 9.374666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 469
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.453125,
+      "epoch": 0.752,
+      "grad_norm": 0.09535495815388843,
+      "kl": 0.082275390625,
+      "learning_rate": 9.373333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 470
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.875,
+      "epoch": 0.7536,
+      "grad_norm": 0.016738803630647392,
+      "kl": 0.08544921875,
+      "learning_rate": 9.372e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 471
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.265625,
+      "epoch": 0.7552,
+      "grad_norm": 0.00973324722077291,
+      "kl": 0.088134765625,
+      "learning_rate": 9.370666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 472
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.671875,
+      "epoch": 0.7568,
+      "grad_norm": 0.007584974577611632,
+      "kl": 0.08251953125,
+      "learning_rate": 9.369333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 473
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.421875,
+      "epoch": 0.7584,
+      "grad_norm": 10.483484616854385,
+      "kl": 0.0927734375,
+      "learning_rate": 9.368e-07,
+      "loss": 0.0001,
+      "reward": 1.640625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.640625,
+      "step": 474
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.78125,
+      "epoch": 0.76,
+      "grad_norm": 2.8402117800611832,
+      "kl": 0.090087890625,
+      "learning_rate": 9.366666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.40625,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.40625,
+      "step": 475
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.9375,
+      "epoch": 0.7616,
+      "grad_norm": 0.007574305426748874,
+      "kl": 0.0771484375,
+      "learning_rate": 9.365333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 476
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.703125,
+      "epoch": 0.7632,
+      "grad_norm": 8.42628141692847,
+      "kl": 0.080078125,
+      "learning_rate": 9.363999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.609375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.609375,
+      "step": 477
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.015625,
+      "epoch": 0.7648,
+      "grad_norm": 0.009245431988300084,
+      "kl": 0.08056640625,
+      "learning_rate": 9.362666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 478
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.640625,
+      "epoch": 0.7664,
+      "grad_norm": 0.006803227511209509,
+      "kl": 0.07763671875,
+      "learning_rate": 9.361333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 479
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.015625,
+      "epoch": 0.768,
+      "grad_norm": 0.017376853421611826,
+      "kl": 0.08984375,
+      "learning_rate": 9.36e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 480
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 141.8125,
+      "epoch": 0.7696,
+      "grad_norm": 45.34387535580377,
+      "kl": 0.069580078125,
+      "learning_rate": 9.358666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.6875,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.6875,
+      "step": 481
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.46875,
+      "epoch": 0.7712,
+      "grad_norm": 0.00889298805212401,
+      "kl": 0.073486328125,
+      "learning_rate": 9.357333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 482
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.25,
+      "epoch": 0.7728,
+      "grad_norm": 0.010209976455841078,
+      "kl": 0.080810546875,
+      "learning_rate": 9.356e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 483
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.921875,
+      "epoch": 0.7744,
+      "grad_norm": 0.01027152197609924,
+      "kl": 0.07861328125,
+      "learning_rate": 9.354666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 484
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.078125,
+      "epoch": 0.776,
+      "grad_norm": 0.01457692453477796,
+      "kl": 0.07421875,
+      "learning_rate": 9.353333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 485
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.734375,
+      "epoch": 0.7776,
+      "grad_norm": 0.006939384002479591,
+      "kl": 0.080078125,
+      "learning_rate": 9.352e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 486
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.71875,
+      "epoch": 0.7792,
+      "grad_norm": 0.012253837962262744,
+      "kl": 0.080322265625,
+      "learning_rate": 9.350666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 487
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.546875,
+      "epoch": 0.7808,
+      "grad_norm": 11.564755846605285,
+      "kl": 0.07763671875,
+      "learning_rate": 9.349333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.6875,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.6875,
+      "step": 488
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.765625,
+      "epoch": 0.7824,
+      "grad_norm": 0.00720694679097634,
+      "kl": 0.07763671875,
+      "learning_rate": 9.347999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 489
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.59375,
+      "epoch": 0.784,
+      "grad_norm": 0.007147781929019379,
+      "kl": 0.0732421875,
+      "learning_rate": 9.346666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 490
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.734375,
+      "epoch": 0.7856,
+      "grad_norm": 0.02885860390489812,
+      "kl": 0.0732421875,
+      "learning_rate": 9.345333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 491
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 143.046875,
+      "epoch": 0.7872,
+      "grad_norm": 0.010424150425107442,
+      "kl": 0.080810546875,
+      "learning_rate": 9.344e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 492
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 141.109375,
+      "epoch": 0.7888,
+      "grad_norm": 0.007034525414946958,
+      "kl": 0.0751953125,
+      "learning_rate": 9.342666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 493
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 143.34375,
+      "epoch": 0.7904,
+      "grad_norm": 0.3653431813899045,
+      "kl": 0.073486328125,
+      "learning_rate": 9.341333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.125,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.125,
+      "step": 494
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 143.0625,
+      "epoch": 0.792,
+      "grad_norm": 1.2098862544793907,
+      "kl": 0.0732421875,
+      "learning_rate": 9.34e-07,
+      "loss": 0.0001,
+      "reward": 1.71875,
+      "reward_std": 0.0883883461356163,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.734375,
+      "step": 495
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.6875,
+      "epoch": 0.7936,
+      "grad_norm": 0.011433607125058016,
+      "kl": 0.079345703125,
+      "learning_rate": 9.338666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 496
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 145.109375,
+      "epoch": 0.7952,
+      "grad_norm": 1.31455358906158,
+      "kl": 0.07275390625,
+      "learning_rate": 9.337333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.59375,
+      "reward_std": 0.0883883461356163,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.609375,
+      "step": 497
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.984375,
+      "epoch": 0.7968,
+      "grad_norm": 0.008337739486346408,
+      "kl": 0.07666015625,
+      "learning_rate": 9.335999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 498
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.90625,
+      "epoch": 0.7984,
+      "grad_norm": 0.01208970274606681,
+      "kl": 0.0751953125,
+      "learning_rate": 9.334666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 499
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.28125,
+      "epoch": 0.8,
+      "grad_norm": 9.397594539958902,
+      "kl": 0.07275390625,
+      "learning_rate": 9.333333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.328125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.328125,
+      "step": 500
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.0625,
+      "epoch": 0.8016,
+      "grad_norm": 0.011396825794678173,
+      "kl": 0.08984375,
+      "learning_rate": 9.332e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 501
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.609375,
+      "epoch": 0.8032,
+      "grad_norm": 12.652928422863626,
+      "kl": 0.07861328125,
+      "learning_rate": 9.330666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.640625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.640625,
+      "step": 502
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.1875,
+      "epoch": 0.8048,
+      "grad_norm": 0.010090360503890044,
+      "kl": 0.082275390625,
+      "learning_rate": 9.329333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 503
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.328125,
+      "epoch": 0.8064,
+      "grad_norm": 0.013353241546864358,
+      "kl": 0.082763671875,
+      "learning_rate": 9.327999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 504
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.5625,
+      "epoch": 0.808,
+      "grad_norm": 0.007500166743914235,
+      "kl": 0.08203125,
+      "learning_rate": 9.326666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 505
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.6875,
+      "epoch": 0.8096,
+      "grad_norm": 0.012267642854857047,
+      "kl": 0.08935546875,
+      "learning_rate": 9.325333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 506
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.390625,
+      "epoch": 0.8112,
+      "grad_norm": 0.009242560555818569,
+      "kl": 0.090087890625,
+      "learning_rate": 9.324e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 507
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.125,
+      "epoch": 0.8128,
+      "grad_norm": 0.010439640191749318,
+      "kl": 0.096435546875,
+      "learning_rate": 9.322666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 508
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.640625,
+      "epoch": 0.8144,
+      "grad_norm": 0.006901283564516389,
+      "kl": 0.080322265625,
+      "learning_rate": 9.321333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 509
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.0,
+      "epoch": 0.816,
+      "grad_norm": 0.01565097179181099,
+      "kl": 0.091064453125,
+      "learning_rate": 9.32e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 510
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.609375,
+      "epoch": 0.8176,
+      "grad_norm": 0.01243992338556979,
+      "kl": 0.08740234375,
+      "learning_rate": 9.318666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 511
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.578125,
+      "epoch": 0.8192,
+      "grad_norm": 0.00897729582528449,
+      "kl": 0.08203125,
+      "learning_rate": 9.317333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 512
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.734375,
+      "epoch": 0.8208,
+      "grad_norm": 0.00608253605834077,
+      "kl": 0.082275390625,
+      "learning_rate": 9.315999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 513
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.84375,
+      "epoch": 0.8224,
+      "grad_norm": 22.061425997474164,
+      "kl": 0.085693359375,
+      "learning_rate": 9.314666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.4375,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.4375,
+      "step": 514
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.96875,
+      "epoch": 0.824,
+      "grad_norm": 1.5102218233410352,
+      "kl": 0.08837890625,
+      "learning_rate": 9.313333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 515
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.75,
+      "epoch": 0.8256,
+      "grad_norm": 0.016642523461715464,
+      "kl": 0.093505859375,
+      "learning_rate": 9.312e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 516
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.359375,
+      "epoch": 0.8272,
+      "grad_norm": 0.009967690314660823,
+      "kl": 0.092041015625,
+      "learning_rate": 9.310666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 517
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.5,
+      "epoch": 0.8288,
+      "grad_norm": 10.82297524980995,
+      "kl": 0.0849609375,
+      "learning_rate": 9.309333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.46875,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.46875,
+      "step": 518
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.015625,
+      "epoch": 0.8304,
+      "grad_norm": 0.007529458889953791,
+      "kl": 0.083984375,
+      "learning_rate": 9.307999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 519
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.09375,
+      "epoch": 0.832,
+      "grad_norm": 0.007613810163829332,
+      "kl": 0.08935546875,
+      "learning_rate": 9.306666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 520
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.40625,
+      "epoch": 0.8336,
+      "grad_norm": 6.634973023830634,
+      "kl": 0.08740234375,
+      "learning_rate": 9.305333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.71875,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.71875,
+      "step": 521
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.15625,
+      "epoch": 0.8352,
+      "grad_norm": 0.007760307480083712,
+      "kl": 0.08984375,
+      "learning_rate": 9.303999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 522
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.203125,
+      "epoch": 0.8368,
+      "grad_norm": 9.71634637235827,
+      "kl": 0.0849609375,
+      "learning_rate": 9.302666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.71875,
+      "reward_std": 0.0883883461356163,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.734375,
+      "step": 523
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.734375,
+      "epoch": 0.8384,
+      "grad_norm": 0.006852825645514184,
+      "kl": 0.0908203125,
+      "learning_rate": 9.301333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 524
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.984375,
+      "epoch": 0.84,
+      "grad_norm": 0.00859685182559928,
+      "kl": 0.0888671875,
+      "learning_rate": 9.3e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 525
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.71875,
+      "epoch": 0.8416,
+      "grad_norm": 1.6072389625445416,
+      "kl": 0.101806640625,
+      "learning_rate": 9.298666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 526
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.625,
+      "epoch": 0.8432,
+      "grad_norm": 0.0065853224034526005,
+      "kl": 0.079345703125,
+      "learning_rate": 9.297333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 527
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.140625,
+      "epoch": 0.8448,
+      "grad_norm": 7.823922137139245,
+      "kl": 0.091552734375,
+      "learning_rate": 9.296e-07,
+      "loss": 0.0001,
+      "reward": 1.703125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.703125,
+      "step": 528
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.109375,
+      "epoch": 0.8464,
+      "grad_norm": 16.56175845728846,
+      "kl": 0.092041015625,
+      "learning_rate": 9.294666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.65625,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.65625,
+      "step": 529
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.65625,
+      "epoch": 0.848,
+      "grad_norm": 23.2321926461147,
+      "kl": 0.10107421875,
+      "learning_rate": 9.293333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.703125,
+      "reward_std": 0.12255740165710449,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.703125,
+      "step": 530
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 103.625,
+      "epoch": 0.8496,
+      "grad_norm": 0.008255385122114534,
+      "kl": 0.0986328125,
+      "learning_rate": 9.292e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 531
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.421875,
+      "epoch": 0.8512,
+      "grad_norm": 18.322948033883698,
+      "kl": 0.1123046875,
+      "learning_rate": 9.290666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.828125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.828125,
+      "step": 532
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 100.59375,
+      "epoch": 0.8528,
+      "grad_norm": 0.007904726004971654,
+      "kl": 0.106201171875,
+      "learning_rate": 9.289333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 533
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.84375,
+      "epoch": 0.8544,
+      "grad_norm": 13.782328020764494,
+      "kl": 0.105712890625,
+      "learning_rate": 9.287999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.515625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.515625,
+      "step": 534
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 103.328125,
+      "epoch": 0.856,
+      "grad_norm": 0.016121488551105352,
+      "kl": 0.094970703125,
+      "learning_rate": 9.286666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 535
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 103.0625,
+      "epoch": 0.8576,
+      "grad_norm": 10.538685148426481,
+      "kl": 0.115966796875,
+      "learning_rate": 9.285333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.640625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.640625,
+      "step": 536
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.203125,
+      "epoch": 0.8592,
+      "grad_norm": 0.008633595085422474,
+      "kl": 0.107666015625,
+      "learning_rate": 9.284e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 537
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 99.125,
+      "epoch": 0.8608,
+      "grad_norm": 0.03588345143252725,
+      "kl": 0.118408203125,
+      "learning_rate": 9.282666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 538
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 104.484375,
+      "epoch": 0.8624,
+      "grad_norm": 9.858938021419759,
+      "kl": 0.10986328125,
+      "learning_rate": 9.281333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.421875,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.421875,
+      "step": 539
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 97.96875,
+      "epoch": 0.864,
+      "grad_norm": 0.012770267108616546,
+      "kl": 0.108642578125,
+      "learning_rate": 9.28e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 540
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 96.1875,
+      "epoch": 0.8656,
+      "grad_norm": 0.010036750278493605,
+      "kl": 0.114013671875,
+      "learning_rate": 9.278666666666665e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 541
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 101.203125,
+      "epoch": 0.8672,
+      "grad_norm": 15.679536652309892,
+      "kl": 0.119140625,
+      "learning_rate": 9.277333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.421875,
+      "reward_std": 0.1804211586713791,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.421875,
+      "step": 542
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 99.828125,
+      "epoch": 0.8688,
+      "grad_norm": 0.01574440718005705,
+      "kl": 0.10107421875,
+      "learning_rate": 9.275999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 543
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 101.03125,
+      "epoch": 0.8704,
+      "grad_norm": 3.3511958230389225,
+      "kl": 0.29736328125,
+      "learning_rate": 9.274666666666666e-07,
+      "loss": 0.0003,
+      "reward": 1.484375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.484375,
+      "step": 544
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 100.484375,
+      "epoch": 0.872,
+      "grad_norm": 4.635822919807855,
+      "kl": 0.115966796875,
+      "learning_rate": 9.273333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 545
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 96.578125,
+      "epoch": 0.8736,
+      "grad_norm": 4.7058051484017085,
+      "kl": 0.122314453125,
+      "learning_rate": 9.272e-07,
+      "loss": 0.0001,
+      "reward": 1.28125,
+      "reward_std": 0.0883883461356163,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.28125,
+      "step": 546
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 99.515625,
+      "epoch": 0.8752,
+      "grad_norm": 98.71315805275106,
+      "kl": 0.108154296875,
+      "learning_rate": 9.270666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.59375,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.59375,
+      "step": 547
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 101.921875,
+      "epoch": 0.8768,
+      "grad_norm": 0.011703016032048912,
+      "kl": 0.109619140625,
+      "learning_rate": 9.269333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 548
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 103.0625,
+      "epoch": 0.8784,
+      "grad_norm": 0.012122322520249933,
+      "kl": 0.121337890625,
+      "learning_rate": 9.268e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 549
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 102.953125,
+      "epoch": 0.88,
+      "grad_norm": 0.007263075292201055,
+      "kl": 0.0986328125,
+      "learning_rate": 9.266666666666665e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 550
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 99.375,
+      "epoch": 0.8816,
+      "grad_norm": 39.082068694328115,
+      "kl": 0.110107421875,
+      "learning_rate": 9.265333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.46875,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.46875,
+      "step": 551
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 104.921875,
+      "epoch": 0.8832,
+      "grad_norm": 6.846345650687952,
+      "kl": 0.093994140625,
+      "learning_rate": 9.263999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.53125,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.53125,
+      "step": 552
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.828125,
+      "epoch": 0.8848,
+      "grad_norm": 4.7833822414228715,
+      "kl": 0.096435546875,
+      "learning_rate": 9.262666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 553
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.5,
+      "epoch": 0.8864,
+      "grad_norm": 0.013586207429355549,
+      "kl": 0.093505859375,
+      "learning_rate": 9.261333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 554
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 104.75,
+      "epoch": 0.888,
+      "grad_norm": 0.007183472017523426,
+      "kl": 0.104248046875,
+      "learning_rate": 9.26e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 555
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.0,
+      "epoch": 0.8896,
+      "grad_norm": 5.65601012982691,
+      "kl": 0.107421875,
+      "learning_rate": 9.258666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.71875,
+      "reward_std": 0.0883883461356163,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.734375,
+      "step": 556
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.234375,
+      "epoch": 0.8912,
+      "grad_norm": 0.007356856657367931,
+      "kl": 0.0927734375,
+      "learning_rate": 9.257333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 557
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.515625,
+      "epoch": 0.8928,
+      "grad_norm": 0.00821578745312473,
+      "kl": 0.096923828125,
+      "learning_rate": 9.256e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 558
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.046875,
+      "epoch": 0.8944,
+      "grad_norm": 0.009303041045393238,
+      "kl": 0.093017578125,
+      "learning_rate": 9.254666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 559
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.984375,
+      "epoch": 0.896,
+      "grad_norm": 0.007948402336780952,
+      "kl": 0.091796875,
+      "learning_rate": 9.253333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 560
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.9375,
+      "epoch": 0.8976,
+      "grad_norm": 16.115683127942994,
+      "kl": 0.1005859375,
+      "learning_rate": 9.251999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.71875,
+      "reward_std": 0.1293872892856598,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.71875,
+      "step": 561
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.515625,
+      "epoch": 0.8992,
+      "grad_norm": 0.008190451727009962,
+      "kl": 0.086181640625,
+      "learning_rate": 9.250666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 562
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.84375,
+      "epoch": 0.9008,
+      "grad_norm": 0.01838192896264667,
+      "kl": 0.093994140625,
+      "learning_rate": 9.249333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 563
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.921875,
+      "epoch": 0.9024,
+      "grad_norm": 21.590534869408724,
+      "kl": 0.0908203125,
+      "learning_rate": 9.247999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.703125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.703125,
+      "step": 564
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.546875,
+      "epoch": 0.904,
+      "grad_norm": 0.010677505613973985,
+      "kl": 0.08984375,
+      "learning_rate": 9.246666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 565
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.65625,
+      "epoch": 0.9056,
+      "grad_norm": 0.007526431795104154,
+      "kl": 0.090087890625,
+      "learning_rate": 9.245333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 566
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.5625,
+      "epoch": 0.9072,
+      "grad_norm": 0.006725934779226597,
+      "kl": 0.0908203125,
+      "learning_rate": 9.244e-07,
+      "loss": 0.0001,
+      "reward": 1.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.0,
+      "step": 567
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.25,
+      "epoch": 0.9088,
+      "grad_norm": 0.008062201355325944,
+      "kl": 0.08642578125,
+      "learning_rate": 9.242666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 568
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.671875,
+      "epoch": 0.9104,
+      "grad_norm": 0.008940567696154457,
+      "kl": 0.093017578125,
+      "learning_rate": 9.241333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 569
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.265625,
+      "epoch": 0.912,
+      "grad_norm": 0.007768704446520214,
+      "kl": 0.08837890625,
+      "learning_rate": 9.24e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 570
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.25,
+      "epoch": 0.9136,
+      "grad_norm": 14.042245206344234,
+      "kl": 0.08349609375,
+      "learning_rate": 9.238666666666665e-07,
+      "loss": 0.0001,
+      "reward": 1.828125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.828125,
+      "step": 571
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.84375,
+      "epoch": 0.9152,
+      "grad_norm": 10.4135986378287,
+      "kl": 0.110107421875,
+      "learning_rate": 9.237333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.359375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.359375,
+      "step": 572
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.40625,
+      "epoch": 0.9168,
+      "grad_norm": 15.292060954166335,
+      "kl": 0.081787109375,
+      "learning_rate": 9.235999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.71875,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.71875,
+      "step": 573
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.421875,
+      "epoch": 0.9184,
+      "grad_norm": 0.009432213349255503,
+      "kl": 0.0869140625,
+      "learning_rate": 9.234666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 574
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.71875,
+      "epoch": 0.92,
+      "grad_norm": 9.59866501943663,
+      "kl": 0.081787109375,
+      "learning_rate": 9.233333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.71875,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.71875,
+      "step": 575
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.1875,
+      "epoch": 0.9216,
+      "grad_norm": 0.021852022519947747,
+      "kl": 0.092529296875,
+      "learning_rate": 9.232e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 576
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.5,
+      "epoch": 0.9232,
+      "grad_norm": 0.021472091831453374,
+      "kl": 0.084228515625,
+      "learning_rate": 9.230666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 577
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.15625,
+      "epoch": 0.9248,
+      "grad_norm": 0.009167248561355507,
+      "kl": 0.0927734375,
+      "learning_rate": 9.229333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 578
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.140625,
+      "epoch": 0.9264,
+      "grad_norm": 0.00618179030910255,
+      "kl": 0.079833984375,
+      "learning_rate": 9.227999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 579
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.453125,
+      "epoch": 0.928,
+      "grad_norm": 0.010088766341825421,
+      "kl": 0.090576171875,
+      "learning_rate": 9.226666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 580
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.484375,
+      "epoch": 0.9296,
+      "grad_norm": 8.488005541612955,
+      "kl": 0.0888671875,
+      "learning_rate": 9.225333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.4375,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.4375,
+      "step": 581
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.875,
+      "epoch": 0.9312,
+      "grad_norm": 1.6179045369135356,
+      "kl": 0.096435546875,
+      "learning_rate": 9.224e-07,
+      "loss": 0.0001,
+      "reward": 1.484375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.484375,
+      "step": 582
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.9375,
+      "epoch": 0.9328,
+      "grad_norm": 0.009979074882269172,
+      "kl": 0.087890625,
+      "learning_rate": 9.222666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 583
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.734375,
+      "epoch": 0.9344,
+      "grad_norm": 0.016747741666008495,
+      "kl": 0.083984375,
+      "learning_rate": 9.221333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 584
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.90625,
+      "epoch": 0.936,
+      "grad_norm": 6.382805721999982,
+      "kl": 0.08837890625,
+      "learning_rate": 9.22e-07,
+      "loss": 0.0001,
+      "reward": 1.53125,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.53125,
+      "step": 585
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.5625,
+      "epoch": 0.9376,
+      "grad_norm": 8.6868172925895,
+      "kl": 0.095703125,
+      "learning_rate": 9.218666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.78125,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.78125,
+      "step": 586
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.546875,
+      "epoch": 0.9392,
+      "grad_norm": 0.01298522315606386,
+      "kl": 0.092041015625,
+      "learning_rate": 9.217333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 587
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.21875,
+      "epoch": 0.9408,
+      "grad_norm": 0.013904659856585358,
+      "kl": 0.095703125,
+      "learning_rate": 9.215999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 588
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.734375,
+      "epoch": 0.9424,
+      "grad_norm": 0.01008859036897162,
+      "kl": 0.094482421875,
+      "learning_rate": 9.214666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 589
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.921875,
+      "epoch": 0.944,
+      "grad_norm": 0.006641429578305623,
+      "kl": 0.090576171875,
+      "learning_rate": 9.213333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 590
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.9375,
+      "epoch": 0.9456,
+      "grad_norm": 0.026959269027031944,
+      "kl": 0.102294921875,
+      "learning_rate": 9.212e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 591
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.203125,
+      "epoch": 0.9472,
+      "grad_norm": 12.652657997824118,
+      "kl": 0.092529296875,
+      "learning_rate": 9.210666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.859375,
+      "reward_std": 0.1315089538693428,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.859375,
+      "step": 592
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.0,
+      "epoch": 0.9488,
+      "grad_norm": 0.016332708399019506,
+      "kl": 0.087890625,
+      "learning_rate": 9.209333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 593
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.015625,
+      "epoch": 0.9504,
+      "grad_norm": 0.012273661758263447,
+      "kl": 0.095703125,
+      "learning_rate": 9.207999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 594
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.953125,
+      "epoch": 0.952,
+      "grad_norm": 0.007876670363292102,
+      "kl": 0.08935546875,
+      "learning_rate": 9.206666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 595
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.078125,
+      "epoch": 0.9536,
+      "grad_norm": 34.83744401223954,
+      "kl": 0.09521484375,
+      "learning_rate": 9.205333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.78125,
+      "reward_std": 0.10888781771063805,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.78125,
+      "step": 596
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.125,
+      "epoch": 0.9552,
+      "grad_norm": 0.012996779422898166,
+      "kl": 0.094970703125,
+      "learning_rate": 9.203999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 597
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.203125,
+      "epoch": 0.9568,
+      "grad_norm": 0.010196058350637506,
+      "kl": 0.088134765625,
+      "learning_rate": 9.202666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 598
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.046875,
+      "epoch": 0.9584,
+      "grad_norm": 5.998194162639431,
+      "kl": 0.0888671875,
+      "learning_rate": 9.201333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.671875,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.671875,
+      "step": 599
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.625,
+      "epoch": 0.96,
+      "grad_norm": 0.017008360590305832,
+      "kl": 0.086669921875,
+      "learning_rate": 9.2e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 600
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.234375,
+      "epoch": 0.9616,
+      "grad_norm": 8.903989801518309,
+      "kl": 0.104736328125,
+      "learning_rate": 9.198666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.59375,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.59375,
+      "step": 601
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.671875,
+      "epoch": 0.9632,
+      "grad_norm": 0.010147494282267702,
+      "kl": 0.0947265625,
+      "learning_rate": 9.197333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 602
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.203125,
+      "epoch": 0.9648,
+      "grad_norm": 2.9788708269934565,
+      "kl": 0.09423828125,
+      "learning_rate": 9.196e-07,
+      "loss": 0.0001,
+      "reward": 1.65625,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.65625,
+      "step": 603
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.0,
+      "epoch": 0.9664,
+      "grad_norm": 0.00705228338418984,
+      "kl": 0.0859375,
+      "learning_rate": 9.194666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 604
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.53125,
+      "epoch": 0.968,
+      "grad_norm": 0.012608787122895648,
+      "kl": 0.083740234375,
+      "learning_rate": 9.193333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 605
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.046875,
+      "epoch": 0.9696,
+      "grad_norm": 6.246424010427853,
+      "kl": 0.08251953125,
+      "learning_rate": 9.192e-07,
+      "loss": 0.0001,
+      "reward": 1.703125,
+      "reward_std": 0.1804211586713791,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.703125,
+      "step": 606
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.890625,
+      "epoch": 0.9712,
+      "grad_norm": 0.011784730541798844,
+      "kl": 0.1015625,
+      "learning_rate": 9.190666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 607
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.9375,
+      "epoch": 0.9728,
+      "grad_norm": 14.479179294369352,
+      "kl": 0.080322265625,
+      "learning_rate": 9.189333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.46875,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.46875,
+      "step": 608
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.359375,
+      "epoch": 0.9744,
+      "grad_norm": 0.009843165606295673,
+      "kl": 0.089111328125,
+      "learning_rate": 9.187999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 609
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.703125,
+      "epoch": 0.976,
+      "grad_norm": 1.574495135665516,
+      "kl": 0.096435546875,
+      "learning_rate": 9.186666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.390625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.390625,
+      "step": 610
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.8125,
+      "epoch": 0.9776,
+      "grad_norm": 0.008128803021805377,
+      "kl": 0.099609375,
+      "learning_rate": 9.185333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 611
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.296875,
+      "epoch": 0.9792,
+      "grad_norm": 0.014561686019849208,
+      "kl": 0.09765625,
+      "learning_rate": 9.184e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 612
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.140625,
+      "epoch": 0.9808,
+      "grad_norm": 0.015333041812894231,
+      "kl": 0.09423828125,
+      "learning_rate": 9.182666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 613
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 103.40625,
+      "epoch": 0.9824,
+      "grad_norm": 0.025755439097613728,
+      "kl": 0.095703125,
+      "learning_rate": 9.181333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 614
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.546875,
+      "epoch": 0.984,
+      "grad_norm": 18.300256266627095,
+      "kl": 0.1015625,
+      "learning_rate": 9.18e-07,
+      "loss": 0.0001,
+      "reward": 1.609375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.609375,
+      "step": 615
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.46875,
+      "epoch": 0.9856,
+      "grad_norm": 0.00896843326026282,
+      "kl": 0.092041015625,
+      "learning_rate": 9.178666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 616
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.265625,
+      "epoch": 0.9872,
+      "grad_norm": 0.01252241631627694,
+      "kl": 0.094482421875,
+      "learning_rate": 9.177333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 617
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.859375,
+      "epoch": 0.9888,
+      "grad_norm": 8.892769409737369,
+      "kl": 0.0947265625,
+      "learning_rate": 9.175999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.71875,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.71875,
+      "step": 618
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 104.953125,
+      "epoch": 0.9904,
+      "grad_norm": 0.008773539191756562,
+      "kl": 0.080078125,
+      "learning_rate": 9.174666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 619
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.984375,
+      "epoch": 0.992,
+      "grad_norm": 0.014347682884966087,
+      "kl": 0.091796875,
+      "learning_rate": 9.173333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 620
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.046875,
+      "epoch": 0.9936,
+      "grad_norm": 0.009028585613493435,
+      "kl": 0.083740234375,
+      "learning_rate": 9.172e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 621
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.171875,
+      "epoch": 0.9952,
+      "grad_norm": 0.015493117421491277,
+      "kl": 0.091552734375,
+      "learning_rate": 9.170666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 622
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.140625,
+      "epoch": 0.9968,
+      "grad_norm": 0.02873566609485228,
+      "kl": 0.089111328125,
+      "learning_rate": 9.169333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 623
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.328125,
+      "epoch": 0.9984,
+      "grad_norm": 0.00988138454400266,
+      "kl": 0.094482421875,
+      "learning_rate": 9.168e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 624
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.875,
+      "epoch": 1.0,
+      "grad_norm": 15.903574655521382,
+      "kl": 0.127197265625,
+      "learning_rate": 9.166666666666665e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 625
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.53125,
+      "epoch": 1.0016,
+      "grad_norm": 0.0153973232517323,
+      "kl": 0.093017578125,
+      "learning_rate": 9.165333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 626
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.203125,
+      "epoch": 1.0032,
+      "grad_norm": 27.853480125926918,
+      "kl": 0.0986328125,
+      "learning_rate": 9.163999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.515625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.515625,
+      "step": 627
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.734375,
+      "epoch": 1.0048,
+      "grad_norm": 0.008169133327209805,
+      "kl": 0.095458984375,
+      "learning_rate": 9.162666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 628
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.546875,
+      "epoch": 1.0064,
+      "grad_norm": 9.389238023348579,
+      "kl": 0.085693359375,
+      "learning_rate": 9.161333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.859375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.859375,
+      "step": 629
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.640625,
+      "epoch": 1.008,
+      "grad_norm": 0.009568873098318597,
+      "kl": 0.091552734375,
+      "learning_rate": 9.16e-07,
+      "loss": 0.0001,
+      "reward": 1.25,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.25,
+      "step": 630
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.296875,
+      "epoch": 1.0096,
+      "grad_norm": 0.023015037727228932,
+      "kl": 0.0869140625,
+      "learning_rate": 9.158666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 631
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.65625,
+      "epoch": 1.0112,
+      "grad_norm": 0.010883725692610391,
+      "kl": 0.093994140625,
+      "learning_rate": 9.157333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 632
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.5,
+      "epoch": 1.0128,
+      "grad_norm": 0.011819847638913906,
+      "kl": 0.098876953125,
+      "learning_rate": 9.156e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 633
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.234375,
+      "epoch": 1.0144,
+      "grad_norm": 1.8836697048070656,
+      "kl": 0.091796875,
+      "learning_rate": 9.154666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.59375,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.59375,
+      "step": 634
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.890625,
+      "epoch": 1.016,
+      "grad_norm": 0.007593584876298436,
+      "kl": 0.084716796875,
+      "learning_rate": 9.153333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 635
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.140625,
+      "epoch": 1.0176,
+      "grad_norm": 1.949779091837715,
+      "kl": 0.091796875,
+      "learning_rate": 9.151999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.640625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.640625,
+      "step": 636
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.3125,
+      "epoch": 1.0192,
+      "grad_norm": 0.009407097543591775,
+      "kl": 0.09765625,
+      "learning_rate": 9.150666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 637
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.703125,
+      "epoch": 1.0208,
+      "grad_norm": 0.010067788196337373,
+      "kl": 0.091064453125,
+      "learning_rate": 9.149333333333333e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 638
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.34375,
+      "epoch": 1.0224,
+      "grad_norm": 0.008939540440613321,
+      "kl": 0.08984375,
+      "learning_rate": 9.147999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 639
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.09375,
+      "epoch": 1.024,
+      "grad_norm": 7.0528529727936125,
+      "kl": 0.09814453125,
+      "learning_rate": 9.146666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.53125,
+      "reward_std": 0.0883883461356163,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.53125,
+      "step": 640
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.953125,
+      "epoch": 1.0256,
+      "grad_norm": 9.764863220821066,
+      "kl": 0.09375,
+      "learning_rate": 9.145333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.4375,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.4375,
+      "step": 641
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.609375,
+      "epoch": 1.0272,
+      "grad_norm": 0.007928981608718045,
+      "kl": 0.0966796875,
+      "learning_rate": 9.144e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 642
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.109375,
+      "epoch": 1.0288,
+      "grad_norm": 0.006237700337885512,
+      "kl": 0.087646484375,
+      "learning_rate": 9.142666666666667e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 643
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.765625,
+      "epoch": 1.0304,
+      "grad_norm": 0.007288731538982205,
+      "kl": 0.085693359375,
+      "learning_rate": 9.141333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 644
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 149.953125,
+      "epoch": 1.032,
+      "grad_norm": 7.8646477887917525,
+      "kl": 0.102783203125,
+      "learning_rate": 9.14e-07,
+      "loss": 0.0001,
+      "reward": 1.765625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.765625,
+      "step": 645
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 144.703125,
+      "epoch": 1.0336,
+      "grad_norm": 15.18502431713818,
+      "kl": 0.085693359375,
+      "learning_rate": 9.138666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.578125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.578125,
+      "step": 646
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 148.59375,
+      "epoch": 1.0352,
+      "grad_norm": 0.007477887654751052,
+      "kl": 0.090576171875,
+      "learning_rate": 9.137333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 647
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 142.84375,
+      "epoch": 1.0368,
+      "grad_norm": 3.0554743156251627,
+      "kl": 0.086669921875,
+      "learning_rate": 9.135999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.859375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.859375,
+      "step": 648
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 144.390625,
+      "epoch": 1.0384,
+      "grad_norm": 14.906796491194266,
+      "kl": 0.081787109375,
+      "learning_rate": 9.134666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.328125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.328125,
+      "step": 649
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.75,
+      "epoch": 1.04,
+      "grad_norm": 6.7087133879045915,
+      "kl": 0.088134765625,
+      "learning_rate": 9.133333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.671875,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.671875,
+      "step": 650
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 148.34375,
+      "epoch": 1.0416,
+      "grad_norm": 13.727071743511654,
+      "kl": 0.084228515625,
+      "learning_rate": 9.132e-07,
+      "loss": 0.0001,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 651
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 152.890625,
+      "epoch": 1.0432,
+      "grad_norm": 0.025835335836071043,
+      "kl": 0.093017578125,
+      "learning_rate": 9.130666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 652
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 156.046875,
+      "epoch": 1.0448,
+      "grad_norm": 0.013461568017209519,
+      "kl": 0.091796875,
+      "learning_rate": 9.129333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 653
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 141.953125,
+      "epoch": 1.0464,
+      "grad_norm": 0.007820612771851764,
+      "kl": 0.08544921875,
+      "learning_rate": 9.127999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 654
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 150.109375,
+      "epoch": 1.048,
+      "grad_norm": 0.019915684215552235,
+      "kl": 0.085693359375,
+      "learning_rate": 9.126666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 655
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 149.296875,
+      "epoch": 1.0496,
+      "grad_norm": 0.010953682276250648,
+      "kl": 0.09521484375,
+      "learning_rate": 9.125333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 656
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 150.8125,
+      "epoch": 1.0512,
+      "grad_norm": 0.006718192127411936,
+      "kl": 0.08544921875,
+      "learning_rate": 9.123999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 657
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 155.890625,
+      "epoch": 1.0528,
+      "grad_norm": 0.008503792175856518,
+      "kl": 0.088134765625,
+      "learning_rate": 9.122666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 658
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 146.984375,
+      "epoch": 1.0544,
+      "grad_norm": 0.007462932010434386,
+      "kl": 0.083984375,
+      "learning_rate": 9.121333333333333e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 659
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 147.03125,
+      "epoch": 1.056,
+      "grad_norm": 0.008790065023389209,
+      "kl": 0.0947265625,
+      "learning_rate": 9.12e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 660
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 148.609375,
+      "epoch": 1.0576,
+      "grad_norm": 0.015097312129176508,
+      "kl": 0.091552734375,
+      "learning_rate": 9.118666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 661
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 149.8125,
+      "epoch": 1.0592,
+      "grad_norm": 24.404999880481803,
+      "kl": 0.082763671875,
+      "learning_rate": 9.117333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.734375,
+      "reward_std": 0.12255740165710449,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 662
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 150.234375,
+      "epoch": 1.0608,
+      "grad_norm": 0.01063489259355667,
+      "kl": 0.094970703125,
+      "learning_rate": 9.115999999999999e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 663
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 146.453125,
+      "epoch": 1.0624,
+      "grad_norm": 0.03190851292449906,
+      "kl": 0.090087890625,
+      "learning_rate": 9.114666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 664
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 143.328125,
+      "epoch": 1.064,
+      "grad_norm": 7.819303213373155,
+      "kl": 0.099609375,
+      "learning_rate": 9.113333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.6875,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.6875,
+      "step": 665
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 148.765625,
+      "epoch": 1.0656,
+      "grad_norm": 0.008388676637737243,
+      "kl": 0.092529296875,
+      "learning_rate": 9.112e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 666
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 144.421875,
+      "epoch": 1.0672,
+      "grad_norm": 0.00814620422078202,
+      "kl": 0.08544921875,
+      "learning_rate": 9.110666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 667
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 152.953125,
+      "epoch": 1.0688,
+      "grad_norm": 0.007486652782363768,
+      "kl": 0.086181640625,
+      "learning_rate": 9.109333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 668
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 148.890625,
+      "epoch": 1.0704,
+      "grad_norm": 0.02660788912797671,
+      "kl": 0.09033203125,
+      "learning_rate": 9.108e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 669
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 147.171875,
+      "epoch": 1.072,
+      "grad_norm": 0.022295457836772694,
+      "kl": 0.0859375,
+      "learning_rate": 9.106666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 670
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 151.3125,
+      "epoch": 1.0735999999999999,
+      "grad_norm": 3.3651824934607477,
+      "kl": 0.083740234375,
+      "learning_rate": 9.105333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.84375,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.84375,
+      "step": 671
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 142.796875,
+      "epoch": 1.0752,
+      "grad_norm": 3.9446732687532142,
+      "kl": 0.091064453125,
+      "learning_rate": 9.103999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.859375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.859375,
+      "step": 672
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 147.859375,
+      "epoch": 1.0768,
+      "grad_norm": 0.006757641932327476,
+      "kl": 0.083984375,
+      "learning_rate": 9.102666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 673
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 146.421875,
+      "epoch": 1.0784,
+      "grad_norm": 1.2258889314326489,
+      "kl": 0.089111328125,
+      "learning_rate": 9.101333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.640625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.640625,
+      "step": 674
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 142.8125,
+      "epoch": 1.08,
+      "grad_norm": 0.006403515019693041,
+      "kl": 0.07958984375,
+      "learning_rate": 9.1e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 675
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 153.78125,
+      "epoch": 1.0816,
+      "grad_norm": 1.9074272156030492,
+      "kl": 0.0869140625,
+      "learning_rate": 9.098666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.84375,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.84375,
+      "step": 676
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 150.28125,
+      "epoch": 1.0832,
+      "grad_norm": 15.405604192326663,
+      "kl": 0.086669921875,
+      "learning_rate": 9.097333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.671875,
+      "reward_std": 0.10205793008208275,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.671875,
+      "step": 677
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 152.8125,
+      "epoch": 1.0848,
+      "grad_norm": 0.006000020006314772,
+      "kl": 0.082275390625,
+      "learning_rate": 9.095999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 678
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 149.671875,
+      "epoch": 1.0864,
+      "grad_norm": 0.007042713935485231,
+      "kl": 0.099365234375,
+      "learning_rate": 9.094666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 679
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 150.65625,
+      "epoch": 1.088,
+      "grad_norm": 0.008675116856098862,
+      "kl": 0.099365234375,
+      "learning_rate": 9.093333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.25,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.25,
+      "step": 680
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 148.59375,
+      "epoch": 1.0896,
+      "grad_norm": 0.00919456182151678,
+      "kl": 0.096923828125,
+      "learning_rate": 9.092e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 681
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 152.859375,
+      "epoch": 1.0912,
+      "grad_norm": 0.006489228032138443,
+      "kl": 0.098388671875,
+      "learning_rate": 9.090666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 682
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 153.921875,
+      "epoch": 1.0928,
+      "grad_norm": 0.007626927956519109,
+      "kl": 0.098388671875,
+      "learning_rate": 9.089333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 683
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 154.0625,
+      "epoch": 1.0944,
+      "grad_norm": 0.0073014739053036985,
+      "kl": 0.0966796875,
+      "learning_rate": 9.088e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 684
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 156.640625,
+      "epoch": 1.096,
+      "grad_norm": 12.357496144289854,
+      "kl": 0.09423828125,
+      "learning_rate": 9.086666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.609375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.609375,
+      "step": 685
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 156.21875,
+      "epoch": 1.0976,
+      "grad_norm": 0.9775789064477919,
+      "kl": 0.104248046875,
+      "learning_rate": 9.085333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.109375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.125,
+      "step": 686
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 154.796875,
+      "epoch": 1.0992,
+      "grad_norm": 2.2145828829658134,
+      "kl": 0.1171875,
+      "learning_rate": 9.084e-07,
+      "loss": 0.0001,
+      "reward": 1.234375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.25,
+      "step": 687
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 143.78125,
+      "epoch": 1.1008,
+      "grad_norm": 0.008682166024316073,
+      "kl": 0.1044921875,
+      "learning_rate": 9.082666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 688
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 146.8125,
+      "epoch": 1.1024,
+      "grad_norm": 0.007107052971170211,
+      "kl": 0.09912109375,
+      "learning_rate": 9.081333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 689
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 142.4375,
+      "epoch": 1.104,
+      "grad_norm": 0.005823380346526666,
+      "kl": 0.1005859375,
+      "learning_rate": 9.08e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 690
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.90625,
+      "epoch": 1.1056,
+      "grad_norm": 0.008819604324688353,
+      "kl": 0.105712890625,
+      "learning_rate": 9.078666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 691
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.203125,
+      "epoch": 1.1072,
+      "grad_norm": 0.01356851553037738,
+      "kl": 0.1142578125,
+      "learning_rate": 9.077333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 692
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 141.453125,
+      "epoch": 1.1088,
+      "grad_norm": 0.017681476548926486,
+      "kl": 0.10693359375,
+      "learning_rate": 9.075999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 693
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.8125,
+      "epoch": 1.1104,
+      "grad_norm": 0.029424884297323192,
+      "kl": 0.116455078125,
+      "learning_rate": 9.074666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 694
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.5625,
+      "epoch": 1.112,
+      "grad_norm": 0.007954578754188658,
+      "kl": 0.09765625,
+      "learning_rate": 9.073333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 695
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.25,
+      "epoch": 1.1136,
+      "grad_norm": 0.007013130950882506,
+      "kl": 0.11083984375,
+      "learning_rate": 9.072e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 696
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.09375,
+      "epoch": 1.1152,
+      "grad_norm": 0.006510008066989398,
+      "kl": 0.099365234375,
+      "learning_rate": 9.070666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 697
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.421875,
+      "epoch": 1.1168,
+      "grad_norm": 0.0065512334729481755,
+      "kl": 0.103515625,
+      "learning_rate": 9.069333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 698
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.375,
+      "epoch": 1.1184,
+      "grad_norm": 0.09605784688486746,
+      "kl": 0.114501953125,
+      "learning_rate": 9.068e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 699
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.890625,
+      "epoch": 1.12,
+      "grad_norm": 3.3141459844008256,
+      "kl": 0.111083984375,
+      "learning_rate": 9.066666666666665e-07,
+      "loss": 0.0001,
+      "reward": 1.640625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.640625,
+      "step": 700
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.203125,
+      "epoch": 1.1216,
+      "grad_norm": 0.012890970477393255,
+      "kl": 0.103515625,
+      "learning_rate": 9.065333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 701
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.703125,
+      "epoch": 1.1232,
+      "grad_norm": 0.016735174682862522,
+      "kl": 0.1083984375,
+      "learning_rate": 9.063999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 702
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.546875,
+      "epoch": 1.1248,
+      "grad_norm": 9.475417426043316,
+      "kl": 0.12548828125,
+      "learning_rate": 9.062666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.359375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.359375,
+      "step": 703
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.375,
+      "epoch": 1.1264,
+      "grad_norm": 0.009014229359184684,
+      "kl": 0.099365234375,
+      "learning_rate": 9.061333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 704
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.125,
+      "epoch": 1.1280000000000001,
+      "grad_norm": 23.520610935812744,
+      "kl": 0.116943359375,
+      "learning_rate": 9.06e-07,
+      "loss": 0.0001,
+      "reward": 1.453125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.453125,
+      "step": 705
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.5,
+      "epoch": 1.1296,
+      "grad_norm": 0.025936735685880525,
+      "kl": 0.114501953125,
+      "learning_rate": 9.058666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 706
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.40625,
+      "epoch": 1.1312,
+      "grad_norm": 6.64619269167457,
+      "kl": 0.109130859375,
+      "learning_rate": 9.057333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.453125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.453125,
+      "step": 707
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.359375,
+      "epoch": 1.1328,
+      "grad_norm": 5.399922291565474,
+      "kl": 0.121337890625,
+      "learning_rate": 9.056e-07,
+      "loss": 0.0001,
+      "reward": 1.828125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.828125,
+      "step": 708
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.609375,
+      "epoch": 1.1344,
+      "grad_norm": 6.723882078303798,
+      "kl": 0.1064453125,
+      "learning_rate": 9.054666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 709
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.0,
+      "epoch": 1.1360000000000001,
+      "grad_norm": 0.01683115108478079,
+      "kl": 0.1103515625,
+      "learning_rate": 9.053333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 710
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.84375,
+      "epoch": 1.1376,
+      "grad_norm": 16.56666318837712,
+      "kl": 0.09619140625,
+      "learning_rate": 9.051999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.765625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.765625,
+      "step": 711
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.359375,
+      "epoch": 1.1392,
+      "grad_norm": 0.00902697690577465,
+      "kl": 0.1123046875,
+      "learning_rate": 9.050666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 712
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.78125,
+      "epoch": 1.1408,
+      "grad_norm": 0.006993125943284126,
+      "kl": 0.1005859375,
+      "learning_rate": 9.049333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 713
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.875,
+      "epoch": 1.1424,
+      "grad_norm": 0.007356421972606694,
+      "kl": 0.10693359375,
+      "learning_rate": 9.048e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 714
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.03125,
+      "epoch": 1.144,
+      "grad_norm": 0.008270666211691296,
+      "kl": 0.10693359375,
+      "learning_rate": 9.046666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 715
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.828125,
+      "epoch": 1.1456,
+      "grad_norm": 0.011092904406612614,
+      "kl": 0.112060546875,
+      "learning_rate": 9.045333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 716
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.40625,
+      "epoch": 1.1472,
+      "grad_norm": 0.006690205149328608,
+      "kl": 0.1083984375,
+      "learning_rate": 9.044e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 717
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.640625,
+      "epoch": 1.1488,
+      "grad_norm": 0.012166041978792308,
+      "kl": 0.1044921875,
+      "learning_rate": 9.042666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 718
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.96875,
+      "epoch": 1.1504,
+      "grad_norm": 0.009054347179680589,
+      "kl": 0.11083984375,
+      "learning_rate": 9.041333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 719
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.9375,
+      "epoch": 1.152,
+      "grad_norm": 0.008709921837647239,
+      "kl": 0.11474609375,
+      "learning_rate": 9.039999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.125,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.125,
+      "step": 720
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.515625,
+      "epoch": 1.1536,
+      "grad_norm": 0.006832349459931436,
+      "kl": 0.104248046875,
+      "learning_rate": 9.038666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 721
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.875,
+      "epoch": 1.1552,
+      "grad_norm": 0.00652521162426839,
+      "kl": 0.087646484375,
+      "learning_rate": 9.037333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 722
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.53125,
+      "epoch": 1.1568,
+      "grad_norm": 0.007037644822808444,
+      "kl": 0.1025390625,
+      "learning_rate": 9.035999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 723
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.890625,
+      "epoch": 1.1584,
+      "grad_norm": 0.01568661842664837,
+      "kl": 0.110107421875,
+      "learning_rate": 9.034666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 724
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.421875,
+      "epoch": 1.16,
+      "grad_norm": 2.4675426460264847,
+      "kl": 0.10302734375,
+      "learning_rate": 9.033333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.609375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.625,
+      "step": 725
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.75,
+      "epoch": 1.1616,
+      "grad_norm": 0.005951896102330027,
+      "kl": 0.09326171875,
+      "learning_rate": 9.032e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 726
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.75,
+      "epoch": 1.1632,
+      "grad_norm": 0.007467806222096547,
+      "kl": 0.10595703125,
+      "learning_rate": 9.030666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 727
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.453125,
+      "epoch": 1.1648,
+      "grad_norm": 0.005586447517572303,
+      "kl": 0.10400390625,
+      "learning_rate": 9.029333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 728
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.1875,
+      "epoch": 1.1663999999999999,
+      "grad_norm": 0.008559954888595884,
+      "kl": 0.094970703125,
+      "learning_rate": 9.028e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 729
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.609375,
+      "epoch": 1.168,
+      "grad_norm": 0.007951991610988375,
+      "kl": 0.117431640625,
+      "learning_rate": 9.026666666666665e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 730
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.78125,
+      "epoch": 1.1696,
+      "grad_norm": 0.019972587312298826,
+      "kl": 0.115478515625,
+      "learning_rate": 9.025333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 731
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.859375,
+      "epoch": 1.1712,
+      "grad_norm": 0.006653402629927198,
+      "kl": 0.104736328125,
+      "learning_rate": 9.023999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.125,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.125,
+      "step": 732
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.578125,
+      "epoch": 1.1728,
+      "grad_norm": 0.00790332330168916,
+      "kl": 0.09619140625,
+      "learning_rate": 9.022666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.25,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.25,
+      "step": 733
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.859375,
+      "epoch": 1.1743999999999999,
+      "grad_norm": 0.006041930958063976,
+      "kl": 0.0947265625,
+      "learning_rate": 9.021333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 734
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.234375,
+      "epoch": 1.176,
+      "grad_norm": 0.0102622034612596,
+      "kl": 0.107666015625,
+      "learning_rate": 9.02e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 735
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.59375,
+      "epoch": 1.1776,
+      "grad_norm": 45.73213503112026,
+      "kl": 0.1005859375,
+      "learning_rate": 9.018666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.640625,
+      "reward_std": 0.12255740165710449,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.640625,
+      "step": 736
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.140625,
+      "epoch": 1.1792,
+      "grad_norm": 0.008324174977812574,
+      "kl": 0.0947265625,
+      "learning_rate": 9.017333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 737
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.5,
+      "epoch": 1.1808,
+      "grad_norm": 22.171944984443087,
+      "kl": 0.114990234375,
+      "learning_rate": 9.015999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.453125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.453125,
+      "step": 738
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.078125,
+      "epoch": 1.1824,
+      "grad_norm": 0.008682434349016327,
+      "kl": 0.097900390625,
+      "learning_rate": 9.014666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 739
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.109375,
+      "epoch": 1.184,
+      "grad_norm": 22.14015861869134,
+      "kl": 0.10400390625,
+      "learning_rate": 9.013333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.390625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.390625,
+      "step": 740
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.453125,
+      "epoch": 1.1856,
+      "grad_norm": 0.02262182977863552,
+      "kl": 0.10302734375,
+      "learning_rate": 9.011999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 741
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.59375,
+      "epoch": 1.1872,
+      "grad_norm": 0.01310829305271454,
+      "kl": 0.098388671875,
+      "learning_rate": 9.010666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 742
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.703125,
+      "epoch": 1.1888,
+      "grad_norm": 0.013273435213056187,
+      "kl": 0.0927734375,
+      "learning_rate": 9.009333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 743
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.375,
+      "epoch": 1.1904,
+      "grad_norm": 0.008150753501788937,
+      "kl": 0.097900390625,
+      "learning_rate": 9.008e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 744
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.78125,
+      "epoch": 1.192,
+      "grad_norm": 0.010918184765140384,
+      "kl": 0.09521484375,
+      "learning_rate": 9.006666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 745
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.0625,
+      "epoch": 1.1936,
+      "grad_norm": 0.00760190953232364,
+      "kl": 0.091064453125,
+      "learning_rate": 9.005333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 746
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.71875,
+      "epoch": 1.1952,
+      "grad_norm": 0.006232547725978523,
+      "kl": 0.092041015625,
+      "learning_rate": 9.004e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 747
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.65625,
+      "epoch": 1.1968,
+      "grad_norm": 17.63786492710451,
+      "kl": 0.09033203125,
+      "learning_rate": 9.002666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.578125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.578125,
+      "step": 748
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.171875,
+      "epoch": 1.1984,
+      "grad_norm": 8.19487250572098,
+      "kl": 0.100830078125,
+      "learning_rate": 9.001333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.296875,
+      "reward_std": 0.10205793380737305,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.296875,
+      "step": 749
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.46875,
+      "epoch": 1.2,
+      "grad_norm": 0.0066797752281432055,
+      "kl": 0.103271484375,
+      "learning_rate": 9e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 750
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.1875,
+      "epoch": 1.2016,
+      "grad_norm": 0.009866889380997399,
+      "kl": 0.095947265625,
+      "learning_rate": 8.998666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 751
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.5,
+      "epoch": 1.2032,
+      "grad_norm": 2.3238208948996246,
+      "kl": 0.405029296875,
+      "learning_rate": 8.997333333333333e-07,
+      "loss": 0.0004,
+      "reward": 1.5625,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5625,
+      "step": 752
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.34375,
+      "epoch": 1.2048,
+      "grad_norm": 0.008881466311384336,
+      "kl": 0.102294921875,
+      "learning_rate": 8.995999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 753
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.609375,
+      "epoch": 1.2064,
+      "grad_norm": 4.773002798162429,
+      "kl": 0.352294921875,
+      "learning_rate": 8.994666666666666e-07,
+      "loss": 0.0004,
+      "reward": 1.640625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.640625,
+      "step": 754
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.15625,
+      "epoch": 1.208,
+      "grad_norm": 0.016625218779008296,
+      "kl": 0.0908203125,
+      "learning_rate": 8.993333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 755
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.703125,
+      "epoch": 1.2096,
+      "grad_norm": 0.019626164460015857,
+      "kl": 0.1025390625,
+      "learning_rate": 8.992e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 756
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.28125,
+      "epoch": 1.2112,
+      "grad_norm": 0.02543193106943962,
+      "kl": 0.10107421875,
+      "learning_rate": 8.990666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 757
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.5625,
+      "epoch": 1.2128,
+      "grad_norm": 0.010201143652185305,
+      "kl": 0.107421875,
+      "learning_rate": 8.989333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 758
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.90625,
+      "epoch": 1.2144,
+      "grad_norm": 0.006337832589738464,
+      "kl": 0.0947265625,
+      "learning_rate": 8.988e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 759
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.25,
+      "epoch": 1.216,
+      "grad_norm": 0.006768661664032184,
+      "kl": 0.091796875,
+      "learning_rate": 8.986666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 760
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.59375,
+      "epoch": 1.2176,
+      "grad_norm": 0.00998033780762735,
+      "kl": 0.118408203125,
+      "learning_rate": 8.985333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 761
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.890625,
+      "epoch": 1.2192,
+      "grad_norm": 39.60344878256688,
+      "kl": 0.114013671875,
+      "learning_rate": 8.983999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.65625,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.65625,
+      "step": 762
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.328125,
+      "epoch": 1.2208,
+      "grad_norm": 0.00810536544200961,
+      "kl": 0.097412109375,
+      "learning_rate": 8.982666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 763
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.984375,
+      "epoch": 1.2224,
+      "grad_norm": 0.012628747149710021,
+      "kl": 0.11328125,
+      "learning_rate": 8.981333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 764
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.796875,
+      "epoch": 1.224,
+      "grad_norm": 0.005865056376253888,
+      "kl": 0.0908203125,
+      "learning_rate": 8.98e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 765
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.671875,
+      "epoch": 1.2256,
+      "grad_norm": 0.00761333475598527,
+      "kl": 0.103759765625,
+      "learning_rate": 8.978666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 766
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.5625,
+      "epoch": 1.2272,
+      "grad_norm": 0.007658755629338665,
+      "kl": 0.0966796875,
+      "learning_rate": 8.977333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 767
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.046875,
+      "epoch": 1.2288000000000001,
+      "grad_norm": 0.010000349139834523,
+      "kl": 0.09521484375,
+      "learning_rate": 8.975999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 768
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.03125,
+      "epoch": 1.2304,
+      "grad_norm": 0.01993669996672977,
+      "kl": 0.108642578125,
+      "learning_rate": 8.974666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 769
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.09375,
+      "epoch": 1.232,
+      "grad_norm": 0.010410851502676064,
+      "kl": 0.102294921875,
+      "learning_rate": 8.973333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 770
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.03125,
+      "epoch": 1.2336,
+      "grad_norm": 0.007268011320824007,
+      "kl": 0.108154296875,
+      "learning_rate": 8.972e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 771
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.78125,
+      "epoch": 1.2352,
+      "grad_norm": 0.01208671359894133,
+      "kl": 0.11669921875,
+      "learning_rate": 8.970666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 772
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.0625,
+      "epoch": 1.2368000000000001,
+      "grad_norm": 0.006957834677760351,
+      "kl": 0.09716796875,
+      "learning_rate": 8.969333333333333e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 773
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.171875,
+      "epoch": 1.2384,
+      "grad_norm": 0.00657002186917475,
+      "kl": 0.095703125,
+      "learning_rate": 8.968e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 774
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.1875,
+      "epoch": 1.24,
+      "grad_norm": 0.01190283594330334,
+      "kl": 0.104248046875,
+      "learning_rate": 8.966666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 775
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.234375,
+      "epoch": 1.2416,
+      "grad_norm": 7.944312303631967,
+      "kl": 0.101318359375,
+      "learning_rate": 8.965333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.28125,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.28125,
+      "step": 776
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.640625,
+      "epoch": 1.2432,
+      "grad_norm": 0.012961372237964616,
+      "kl": 0.0986328125,
+      "learning_rate": 8.963999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 777
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.546875,
+      "epoch": 1.2448,
+      "grad_norm": 0.008728740403488155,
+      "kl": 0.093994140625,
+      "learning_rate": 8.962666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 778
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.171875,
+      "epoch": 1.2464,
+      "grad_norm": 0.01645090866795687,
+      "kl": 0.090576171875,
+      "learning_rate": 8.961333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 779
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.5625,
+      "epoch": 1.248,
+      "grad_norm": 0.01714953542307708,
+      "kl": 0.100830078125,
+      "learning_rate": 8.96e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 780
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.84375,
+      "epoch": 1.2496,
+      "grad_norm": 0.011559257349018309,
+      "kl": 0.099365234375,
+      "learning_rate": 8.958666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 781
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.796875,
+      "epoch": 1.2511999999999999,
+      "grad_norm": 0.007882372312793819,
+      "kl": 0.097900390625,
+      "learning_rate": 8.957333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 782
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.09375,
+      "epoch": 1.2528000000000001,
+      "grad_norm": 0.011324233608162074,
+      "kl": 0.09716796875,
+      "learning_rate": 8.955999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 783
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.328125,
+      "epoch": 1.2544,
+      "grad_norm": 45.97087749283503,
+      "kl": 0.09765625,
+      "learning_rate": 8.954666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 784
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.171875,
+      "epoch": 1.256,
+      "grad_norm": 0.00646735155361448,
+      "kl": 0.092529296875,
+      "learning_rate": 8.953333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 785
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.671875,
+      "epoch": 1.2576,
+      "grad_norm": 18.767612237698795,
+      "kl": 0.09716796875,
+      "learning_rate": 8.951999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.84375,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.84375,
+      "step": 786
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.890625,
+      "epoch": 1.2591999999999999,
+      "grad_norm": 0.018592355624831924,
+      "kl": 0.086669921875,
+      "learning_rate": 8.950666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 787
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.6875,
+      "epoch": 1.2608,
+      "grad_norm": 0.029077548981741252,
+      "kl": 0.09521484375,
+      "learning_rate": 8.949333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 788
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.0,
+      "epoch": 1.2624,
+      "grad_norm": 43.47963907342033,
+      "kl": 0.093994140625,
+      "learning_rate": 8.948e-07,
+      "loss": 0.0001,
+      "reward": 1.71875,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.71875,
+      "step": 789
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.890625,
+      "epoch": 1.264,
+      "grad_norm": 19.479424254471763,
+      "kl": 0.099609375,
+      "learning_rate": 8.946666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 790
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.34375,
+      "epoch": 1.2656,
+      "grad_norm": 0.006073502140047607,
+      "kl": 0.090576171875,
+      "learning_rate": 8.945333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 791
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.140625,
+      "epoch": 1.2671999999999999,
+      "grad_norm": 0.005846824617619844,
+      "kl": 0.092529296875,
+      "learning_rate": 8.944e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 792
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.734375,
+      "epoch": 1.2688,
+      "grad_norm": 0.012666623747735063,
+      "kl": 0.11083984375,
+      "learning_rate": 8.942666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 793
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.921875,
+      "epoch": 1.2704,
+      "grad_norm": 0.008138505400974218,
+      "kl": 0.097900390625,
+      "learning_rate": 8.941333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 794
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.984375,
+      "epoch": 1.272,
+      "grad_norm": 21.28992593614142,
+      "kl": 0.087646484375,
+      "learning_rate": 8.939999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.84375,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.84375,
+      "step": 795
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.828125,
+      "epoch": 1.2736,
+      "grad_norm": 0.009969240755347844,
+      "kl": 0.096923828125,
+      "learning_rate": 8.938666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 796
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.71875,
+      "epoch": 1.2752,
+      "grad_norm": 0.010759735631953222,
+      "kl": 0.10400390625,
+      "learning_rate": 8.937333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 797
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.875,
+      "epoch": 1.2768,
+      "grad_norm": 0.009981296122954158,
+      "kl": 0.0927734375,
+      "learning_rate": 8.935999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 798
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.265625,
+      "epoch": 1.2784,
+      "grad_norm": 0.008586981643758032,
+      "kl": 0.095947265625,
+      "learning_rate": 8.934666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 799
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.859375,
+      "epoch": 1.28,
+      "grad_norm": 0.006328228289573948,
+      "kl": 0.085205078125,
+      "learning_rate": 8.933333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 800
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.390625,
+      "epoch": 1.2816,
+      "grad_norm": 0.43301368121876505,
+      "kl": 0.095458984375,
+      "learning_rate": 8.932e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 801
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.484375,
+      "epoch": 1.2832,
+      "grad_norm": 0.008983362321650397,
+      "kl": 0.091064453125,
+      "learning_rate": 8.930666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 802
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.171875,
+      "epoch": 1.2848,
+      "grad_norm": 1.9492149110820276,
+      "kl": 0.085693359375,
+      "learning_rate": 8.929333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.484375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.5,
+      "step": 803
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.53125,
+      "epoch": 1.2864,
+      "grad_norm": 0.0071095090410322115,
+      "kl": 0.0947265625,
+      "learning_rate": 8.928e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 804
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.453125,
+      "epoch": 1.288,
+      "grad_norm": 0.009478694830462193,
+      "kl": 0.103515625,
+      "learning_rate": 8.926666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 805
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.390625,
+      "epoch": 1.2896,
+      "grad_norm": 0.011332469454608916,
+      "kl": 0.0869140625,
+      "learning_rate": 8.925333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 806
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.1875,
+      "epoch": 1.2912,
+      "grad_norm": 0.00866410042960962,
+      "kl": 0.088134765625,
+      "learning_rate": 8.923999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 807
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.921875,
+      "epoch": 1.2928,
+      "grad_norm": 0.009288333904917385,
+      "kl": 0.0966796875,
+      "learning_rate": 8.922666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 808
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.109375,
+      "epoch": 1.2944,
+      "grad_norm": 0.005682439487071505,
+      "kl": 0.0859375,
+      "learning_rate": 8.921333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 809
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.84375,
+      "epoch": 1.296,
+      "grad_norm": 0.006832002191755821,
+      "kl": 0.094482421875,
+      "learning_rate": 8.92e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 810
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.765625,
+      "epoch": 1.2976,
+      "grad_norm": 0.011453880407877972,
+      "kl": 0.110595703125,
+      "learning_rate": 8.918666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 811
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.109375,
+      "epoch": 1.2992,
+      "grad_norm": 0.00825878096674971,
+      "kl": 0.098876953125,
+      "learning_rate": 8.917333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 812
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.71875,
+      "epoch": 1.3008,
+      "grad_norm": 0.05141858195154446,
+      "kl": 0.08984375,
+      "learning_rate": 8.915999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 813
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.78125,
+      "epoch": 1.3024,
+      "grad_norm": 0.009077255154466406,
+      "kl": 0.095703125,
+      "learning_rate": 8.914666666666665e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 814
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.328125,
+      "epoch": 1.304,
+      "grad_norm": 0.007970113849291464,
+      "kl": 0.093017578125,
+      "learning_rate": 8.913333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 815
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.96875,
+      "epoch": 1.3056,
+      "grad_norm": 2.075834920136862,
+      "kl": 0.103759765625,
+      "learning_rate": 8.911999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 816
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.703125,
+      "epoch": 1.3072,
+      "grad_norm": 12.145911570233132,
+      "kl": 0.09326171875,
+      "learning_rate": 8.910666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.78125,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.78125,
+      "step": 817
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.953125,
+      "epoch": 1.3088,
+      "grad_norm": 0.011971025394521976,
+      "kl": 0.092529296875,
+      "learning_rate": 8.909333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 818
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.46875,
+      "epoch": 1.3104,
+      "grad_norm": 0.009097659261596709,
+      "kl": 0.0888671875,
+      "learning_rate": 8.908e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 819
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.25,
+      "epoch": 1.312,
+      "grad_norm": 0.00870396194397433,
+      "kl": 0.094970703125,
+      "learning_rate": 8.906666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 820
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.8125,
+      "epoch": 1.3136,
+      "grad_norm": 0.01223422332078379,
+      "kl": 0.08984375,
+      "learning_rate": 8.905333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 821
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.265625,
+      "epoch": 1.3152,
+      "grad_norm": 1.5405911528513196,
+      "kl": 0.0966796875,
+      "learning_rate": 8.904e-07,
+      "loss": 0.0001,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 822
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.015625,
+      "epoch": 1.3168,
+      "grad_norm": 0.008522516258871172,
+      "kl": 0.091064453125,
+      "learning_rate": 8.902666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 823
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.5625,
+      "epoch": 1.3184,
+      "grad_norm": 0.266164024692273,
+      "kl": 0.08837890625,
+      "learning_rate": 8.901333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 824
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.78125,
+      "epoch": 1.32,
+      "grad_norm": 0.006569515927258924,
+      "kl": 0.083984375,
+      "learning_rate": 8.9e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 825
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.03125,
+      "epoch": 1.3216,
+      "grad_norm": 0.01203029112408642,
+      "kl": 0.104248046875,
+      "learning_rate": 8.898666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 826
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.859375,
+      "epoch": 1.3232,
+      "grad_norm": 0.00980755263187315,
+      "kl": 0.101318359375,
+      "learning_rate": 8.897333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 827
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.921875,
+      "epoch": 1.3248,
+      "grad_norm": 9.909432699670496,
+      "kl": 0.098388671875,
+      "learning_rate": 8.895999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.359375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.359375,
+      "step": 828
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.4375,
+      "epoch": 1.3264,
+      "grad_norm": 0.008614179686174133,
+      "kl": 0.09814453125,
+      "learning_rate": 8.894666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 829
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.28125,
+      "epoch": 1.328,
+      "grad_norm": 23.739663667078926,
+      "kl": 0.09814453125,
+      "learning_rate": 8.893333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.640625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.640625,
+      "step": 830
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.15625,
+      "epoch": 1.3296000000000001,
+      "grad_norm": 0.008107344637455794,
+      "kl": 0.1015625,
+      "learning_rate": 8.892e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 831
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.15625,
+      "epoch": 1.3312,
+      "grad_norm": 18.842857370372986,
+      "kl": 0.09814453125,
+      "learning_rate": 8.890666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.515625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.515625,
+      "step": 832
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.46875,
+      "epoch": 1.3328,
+      "grad_norm": 0.007202358696651244,
+      "kl": 0.09228515625,
+      "learning_rate": 8.889333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 833
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 103.5625,
+      "epoch": 1.3344,
+      "grad_norm": 0.0071037628829024695,
+      "kl": 0.1015625,
+      "learning_rate": 8.888e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 834
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.0625,
+      "epoch": 1.336,
+      "grad_norm": 0.007557005137263755,
+      "kl": 0.0966796875,
+      "learning_rate": 8.886666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 835
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.765625,
+      "epoch": 1.3376000000000001,
+      "grad_norm": 6.0228325740184,
+      "kl": 0.107177734375,
+      "learning_rate": 8.885333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.71875,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.71875,
+      "step": 836
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 101.671875,
+      "epoch": 1.3392,
+      "grad_norm": 0.018166673716594414,
+      "kl": 0.115966796875,
+      "learning_rate": 8.883999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 837
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.59375,
+      "epoch": 1.3408,
+      "grad_norm": 15.272170540658392,
+      "kl": 0.09814453125,
+      "learning_rate": 8.882666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 838
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.125,
+      "epoch": 1.3424,
+      "grad_norm": 0.008327452119324777,
+      "kl": 0.100341796875,
+      "learning_rate": 8.881333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 839
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 102.4375,
+      "epoch": 1.3439999999999999,
+      "grad_norm": 0.012511663071686479,
+      "kl": 0.11083984375,
+      "learning_rate": 8.88e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 840
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.578125,
+      "epoch": 1.3456000000000001,
+      "grad_norm": 0.007719419947110325,
+      "kl": 0.099853515625,
+      "learning_rate": 8.878666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 841
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.328125,
+      "epoch": 1.3472,
+      "grad_norm": 0.01007586202530742,
+      "kl": 0.1201171875,
+      "learning_rate": 8.877333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 842
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.5,
+      "epoch": 1.3488,
+      "grad_norm": 2.4483595270997256,
+      "kl": 0.107666015625,
+      "learning_rate": 8.875999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.515625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.515625,
+      "step": 843
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 104.84375,
+      "epoch": 1.3504,
+      "grad_norm": 0.013768860984981602,
+      "kl": 0.10009765625,
+      "learning_rate": 8.874666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 844
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.09375,
+      "epoch": 1.3519999999999999,
+      "grad_norm": 0.00854642093417692,
+      "kl": 0.095458984375,
+      "learning_rate": 8.873333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 845
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 104.5625,
+      "epoch": 1.3536000000000001,
+      "grad_norm": 3.9755555054326766,
+      "kl": 0.113525390625,
+      "learning_rate": 8.872e-07,
+      "loss": 0.0001,
+      "reward": 1.96875,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.96875,
+      "step": 846
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.25,
+      "epoch": 1.3552,
+      "grad_norm": 0.019167074406571012,
+      "kl": 0.100830078125,
+      "learning_rate": 8.870666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.25,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.25,
+      "step": 847
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.90625,
+      "epoch": 1.3568,
+      "grad_norm": 0.011434732268681464,
+      "kl": 0.10009765625,
+      "learning_rate": 8.869333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 848
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.703125,
+      "epoch": 1.3584,
+      "grad_norm": 0.007237256894068788,
+      "kl": 0.099609375,
+      "learning_rate": 8.868e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 849
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 101.5625,
+      "epoch": 1.3599999999999999,
+      "grad_norm": 0.008972535663438252,
+      "kl": 0.1162109375,
+      "learning_rate": 8.866666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 850
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.28125,
+      "epoch": 1.3616,
+      "grad_norm": 0.03071251595281918,
+      "kl": 0.120849609375,
+      "learning_rate": 8.865333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 851
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.65625,
+      "epoch": 1.3632,
+      "grad_norm": 0.007067133008383711,
+      "kl": 0.094970703125,
+      "learning_rate": 8.863999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 852
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.921875,
+      "epoch": 1.3648,
+      "grad_norm": 0.011460731346261558,
+      "kl": 0.103515625,
+      "learning_rate": 8.862666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 853
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.703125,
+      "epoch": 1.3664,
+      "grad_norm": 0.008797112676397201,
+      "kl": 0.1005859375,
+      "learning_rate": 8.861333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 854
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.015625,
+      "epoch": 1.3679999999999999,
+      "grad_norm": 0.007167311712836972,
+      "kl": 0.109619140625,
+      "learning_rate": 8.86e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 855
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.71875,
+      "epoch": 1.3696,
+      "grad_norm": 0.008399036285472256,
+      "kl": 0.10791015625,
+      "learning_rate": 8.858666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 856
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.0,
+      "epoch": 1.3712,
+      "grad_norm": 0.008726530288898462,
+      "kl": 0.09619140625,
+      "learning_rate": 8.857333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 857
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.546875,
+      "epoch": 1.3728,
+      "grad_norm": 8.823463942499817,
+      "kl": 0.099609375,
+      "learning_rate": 8.856e-07,
+      "loss": 0.0001,
+      "reward": 1.578125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.578125,
+      "step": 858
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.15625,
+      "epoch": 1.3744,
+      "grad_norm": 0.00642282813293384,
+      "kl": 0.10498046875,
+      "learning_rate": 8.854666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 859
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.71875,
+      "epoch": 1.376,
+      "grad_norm": 0.014910046395033252,
+      "kl": 0.09716796875,
+      "learning_rate": 8.853333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.25,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.25,
+      "step": 860
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.75,
+      "epoch": 1.3776,
+      "grad_norm": 0.00789570805278616,
+      "kl": 0.10400390625,
+      "learning_rate": 8.851999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 861
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.0625,
+      "epoch": 1.3792,
+      "grad_norm": 0.009708503843419388,
+      "kl": 0.114990234375,
+      "learning_rate": 8.850666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 862
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.953125,
+      "epoch": 1.3808,
+      "grad_norm": 0.008987675338968434,
+      "kl": 0.10009765625,
+      "learning_rate": 8.849333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 863
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.34375,
+      "epoch": 1.3824,
+      "grad_norm": 7.330977890775365,
+      "kl": 0.0966796875,
+      "learning_rate": 8.848e-07,
+      "loss": 0.0001,
+      "reward": 1.4375,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.4375,
+      "step": 864
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.984375,
+      "epoch": 1.384,
+      "grad_norm": 0.02645437771131234,
+      "kl": 0.111328125,
+      "learning_rate": 8.846666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 865
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.9375,
+      "epoch": 1.3856,
+      "grad_norm": 0.009455352132785108,
+      "kl": 0.10498046875,
+      "learning_rate": 8.845333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 866
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.34375,
+      "epoch": 1.3872,
+      "grad_norm": 0.007700257547621883,
+      "kl": 0.101318359375,
+      "learning_rate": 8.844e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 867
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.203125,
+      "epoch": 1.3888,
+      "grad_norm": 0.0072915677792911165,
+      "kl": 0.11328125,
+      "learning_rate": 8.842666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 868
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.28125,
+      "epoch": 1.3904,
+      "grad_norm": 0.009457324729595552,
+      "kl": 0.09814453125,
+      "learning_rate": 8.841333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 869
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.296875,
+      "epoch": 1.392,
+      "grad_norm": 0.009531541058694845,
+      "kl": 0.103515625,
+      "learning_rate": 8.839999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 870
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.59375,
+      "epoch": 1.3936,
+      "grad_norm": 0.00727591773922625,
+      "kl": 0.101318359375,
+      "learning_rate": 8.838666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 871
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.6875,
+      "epoch": 1.3952,
+      "grad_norm": 0.009874183548236097,
+      "kl": 0.09912109375,
+      "learning_rate": 8.837333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 872
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.828125,
+      "epoch": 1.3968,
+      "grad_norm": 0.009720564402489827,
+      "kl": 0.097900390625,
+      "learning_rate": 8.836e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 873
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.46875,
+      "epoch": 1.3984,
+      "grad_norm": 0.0167195011551099,
+      "kl": 0.10693359375,
+      "learning_rate": 8.834666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 874
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.4375,
+      "epoch": 1.4,
+      "grad_norm": 0.00914669842678689,
+      "kl": 0.091552734375,
+      "learning_rate": 8.833333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 875
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.390625,
+      "epoch": 1.4016,
+      "grad_norm": 0.010380859327959588,
+      "kl": 0.101806640625,
+      "learning_rate": 8.832e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 876
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.015625,
+      "epoch": 1.4032,
+      "grad_norm": 0.011589110940673246,
+      "kl": 0.090576171875,
+      "learning_rate": 8.830666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 877
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.0,
+      "epoch": 1.4048,
+      "grad_norm": 0.010828341354014975,
+      "kl": 0.09814453125,
+      "learning_rate": 8.829333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 878
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.6875,
+      "epoch": 1.4064,
+      "grad_norm": 0.011769898157650265,
+      "kl": 0.118408203125,
+      "learning_rate": 8.827999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 879
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.84375,
+      "epoch": 1.408,
+      "grad_norm": 0.009087591072350078,
+      "kl": 0.107177734375,
+      "learning_rate": 8.826666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 880
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.578125,
+      "epoch": 1.4096,
+      "grad_norm": 0.0077615734689265355,
+      "kl": 0.106201171875,
+      "learning_rate": 8.825333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 881
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.0625,
+      "epoch": 1.4112,
+      "grad_norm": 18.01822253841369,
+      "kl": 0.09716796875,
+      "learning_rate": 8.823999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.703125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.703125,
+      "step": 882
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.71875,
+      "epoch": 1.4128,
+      "grad_norm": 0.011411512965323204,
+      "kl": 0.09765625,
+      "learning_rate": 8.822666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 883
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.90625,
+      "epoch": 1.4144,
+      "grad_norm": 0.00968930702921202,
+      "kl": 0.100341796875,
+      "learning_rate": 8.821333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 884
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.71875,
+      "epoch": 1.416,
+      "grad_norm": 27.25212373440593,
+      "kl": 0.089599609375,
+      "learning_rate": 8.82e-07,
+      "loss": 0.0001,
+      "reward": 1.9375,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.9375,
+      "step": 885
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.71875,
+      "epoch": 1.4176,
+      "grad_norm": 0.011797906000422026,
+      "kl": 0.108642578125,
+      "learning_rate": 8.818666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 886
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.140625,
+      "epoch": 1.4192,
+      "grad_norm": 0.008244026310598337,
+      "kl": 0.100830078125,
+      "learning_rate": 8.817333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 887
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.015625,
+      "epoch": 1.4208,
+      "grad_norm": 0.012161028900532233,
+      "kl": 0.093017578125,
+      "learning_rate": 8.816000000000001e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 888
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.828125,
+      "epoch": 1.4224,
+      "grad_norm": 0.009515154743807742,
+      "kl": 0.100830078125,
+      "learning_rate": 8.814666666666665e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 889
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.484375,
+      "epoch": 1.424,
+      "grad_norm": 0.006863683018948688,
+      "kl": 0.0966796875,
+      "learning_rate": 8.813333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 890
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.234375,
+      "epoch": 1.4256,
+      "grad_norm": 0.010339814758920086,
+      "kl": 0.09521484375,
+      "learning_rate": 8.811999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 891
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.515625,
+      "epoch": 1.4272,
+      "grad_norm": 25.707019256583617,
+      "kl": 0.094482421875,
+      "learning_rate": 8.810666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5625,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5625,
+      "step": 892
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.5625,
+      "epoch": 1.4288,
+      "grad_norm": 0.011674473737463654,
+      "kl": 0.10400390625,
+      "learning_rate": 8.809333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 893
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.34375,
+      "epoch": 1.4304000000000001,
+      "grad_norm": 0.02552479194781142,
+      "kl": 0.09228515625,
+      "learning_rate": 8.808e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 894
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.328125,
+      "epoch": 1.432,
+      "grad_norm": 6.9504294349275995,
+      "kl": 0.0908203125,
+      "learning_rate": 8.806666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.859375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.859375,
+      "step": 895
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.984375,
+      "epoch": 1.4336,
+      "grad_norm": 0.009016656704441869,
+      "kl": 0.09130859375,
+      "learning_rate": 8.805333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 896
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.890625,
+      "epoch": 1.4352,
+      "grad_norm": 0.015432241842612724,
+      "kl": 0.086669921875,
+      "learning_rate": 8.804e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 897
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.25,
+      "epoch": 1.4368,
+      "grad_norm": 0.008551775890942998,
+      "kl": 0.086181640625,
+      "learning_rate": 8.802666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 898
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.078125,
+      "epoch": 1.4384000000000001,
+      "grad_norm": 0.006279693966815684,
+      "kl": 0.09765625,
+      "learning_rate": 8.801333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 899
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.3125,
+      "epoch": 1.44,
+      "grad_norm": 0.014695761976683513,
+      "kl": 0.087890625,
+      "learning_rate": 8.799999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 900
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.0625,
+      "epoch": 1.4416,
+      "grad_norm": 2.122088294095446,
+      "kl": 0.095458984375,
+      "learning_rate": 8.798666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.484375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.484375,
+      "step": 901
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.59375,
+      "epoch": 1.4432,
+      "grad_norm": 0.04109373478478731,
+      "kl": 0.104736328125,
+      "learning_rate": 8.797333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 902
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.3125,
+      "epoch": 1.4447999999999999,
+      "grad_norm": 0.007403651476754684,
+      "kl": 0.092529296875,
+      "learning_rate": 8.796e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 903
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.140625,
+      "epoch": 1.4464000000000001,
+      "grad_norm": 5.978350747532322,
+      "kl": 0.085205078125,
+      "learning_rate": 8.794666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.765625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.765625,
+      "step": 904
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.984375,
+      "epoch": 1.448,
+      "grad_norm": 0.006527810592062156,
+      "kl": 0.092041015625,
+      "learning_rate": 8.793333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 905
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.3125,
+      "epoch": 1.4496,
+      "grad_norm": 0.015049416757420258,
+      "kl": 0.10595703125,
+      "learning_rate": 8.792e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 906
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.125,
+      "epoch": 1.4512,
+      "grad_norm": 0.00814146848910921,
+      "kl": 0.090576171875,
+      "learning_rate": 8.790666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 907
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.109375,
+      "epoch": 1.4527999999999999,
+      "grad_norm": 0.009431059667657563,
+      "kl": 0.09814453125,
+      "learning_rate": 8.789333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 908
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.984375,
+      "epoch": 1.4544000000000001,
+      "grad_norm": 0.015109852736911048,
+      "kl": 0.099365234375,
+      "learning_rate": 8.788e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 909
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.65625,
+      "epoch": 1.456,
+      "grad_norm": 0.006894850515148367,
+      "kl": 0.095703125,
+      "learning_rate": 8.786666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 910
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.03125,
+      "epoch": 1.4576,
+      "grad_norm": 0.010569548107209299,
+      "kl": 0.09375,
+      "learning_rate": 8.785333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 911
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.4375,
+      "epoch": 1.4592,
+      "grad_norm": 0.009636026095821273,
+      "kl": 0.080078125,
+      "learning_rate": 8.783999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 912
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.0625,
+      "epoch": 1.4607999999999999,
+      "grad_norm": 5.274983090754911,
+      "kl": 0.09228515625,
+      "learning_rate": 8.782666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.75,
+      "step": 913
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.15625,
+      "epoch": 1.4624,
+      "grad_norm": 0.010658943707061505,
+      "kl": 0.09619140625,
+      "learning_rate": 8.781333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 914
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.421875,
+      "epoch": 1.464,
+      "grad_norm": 0.008691115707950735,
+      "kl": 0.096435546875,
+      "learning_rate": 8.78e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 915
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.4375,
+      "epoch": 1.4656,
+      "grad_norm": 0.006426503218383063,
+      "kl": 0.093505859375,
+      "learning_rate": 8.778666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 916
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.65625,
+      "epoch": 1.4672,
+      "grad_norm": 0.007281403308872408,
+      "kl": 0.09619140625,
+      "learning_rate": 8.777333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 917
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.984375,
+      "epoch": 1.4687999999999999,
+      "grad_norm": 0.009435770769411881,
+      "kl": 0.099609375,
+      "learning_rate": 8.776e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 918
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.046875,
+      "epoch": 1.4704,
+      "grad_norm": 0.007923531732382187,
+      "kl": 0.102783203125,
+      "learning_rate": 8.774666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.25,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.25,
+      "step": 919
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.828125,
+      "epoch": 1.472,
+      "grad_norm": 0.00830695071130712,
+      "kl": 0.10107421875,
+      "learning_rate": 8.773333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 920
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.515625,
+      "epoch": 1.4736,
+      "grad_norm": 0.007059788723138681,
+      "kl": 0.090087890625,
+      "learning_rate": 8.771999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 921
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.390625,
+      "epoch": 1.4752,
+      "grad_norm": 0.007849721693457382,
+      "kl": 0.1064453125,
+      "learning_rate": 8.770666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 922
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.09375,
+      "epoch": 1.4768,
+      "grad_norm": 0.009520152241068468,
+      "kl": 0.102294921875,
+      "learning_rate": 8.769333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 923
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.765625,
+      "epoch": 1.4784,
+      "grad_norm": 0.005991366364087312,
+      "kl": 0.08935546875,
+      "learning_rate": 8.768e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 924
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.625,
+      "epoch": 1.48,
+      "grad_norm": 0.13804959310898648,
+      "kl": 0.1064453125,
+      "learning_rate": 8.766666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 925
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.953125,
+      "epoch": 1.4816,
+      "grad_norm": 0.0054856829483171015,
+      "kl": 0.0859375,
+      "learning_rate": 8.765333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 926
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.40625,
+      "epoch": 1.4832,
+      "grad_norm": 0.02310485709938161,
+      "kl": 0.084716796875,
+      "learning_rate": 8.763999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 927
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.765625,
+      "epoch": 1.4848,
+      "grad_norm": 0.006352216655375585,
+      "kl": 0.087646484375,
+      "learning_rate": 8.762666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.125,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.125,
+      "step": 928
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.5,
+      "epoch": 1.4864,
+      "grad_norm": 23.98074231850722,
+      "kl": 0.100341796875,
+      "learning_rate": 8.761333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.359375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.359375,
+      "step": 929
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.109375,
+      "epoch": 1.488,
+      "grad_norm": 0.00995453017780271,
+      "kl": 0.1083984375,
+      "learning_rate": 8.76e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 930
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.0,
+      "epoch": 1.4896,
+      "grad_norm": 0.013939766943273703,
+      "kl": 0.09423828125,
+      "learning_rate": 8.758666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 931
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.953125,
+      "epoch": 1.4912,
+      "grad_norm": 0.007260981211265019,
+      "kl": 0.092529296875,
+      "learning_rate": 8.757333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 932
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.40625,
+      "epoch": 1.4928,
+      "grad_norm": 0.005702579312528671,
+      "kl": 0.087646484375,
+      "learning_rate": 8.756e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 933
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.078125,
+      "epoch": 1.4944,
+      "grad_norm": 0.005936178126545748,
+      "kl": 0.080322265625,
+      "learning_rate": 8.754666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 934
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.34375,
+      "epoch": 1.496,
+      "grad_norm": 0.03438778030719355,
+      "kl": 0.086669921875,
+      "learning_rate": 8.753333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 935
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.015625,
+      "epoch": 1.4976,
+      "grad_norm": 0.006587709175816995,
+      "kl": 0.093994140625,
+      "learning_rate": 8.751999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 936
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 141.4375,
+      "epoch": 1.4992,
+      "grad_norm": 52.46533985611934,
+      "kl": 0.094482421875,
+      "learning_rate": 8.750666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.609375,
+      "reward_std": 0.13258251920342445,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.625,
+      "step": 937
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.3125,
+      "epoch": 1.5008,
+      "grad_norm": 0.006420689210629613,
+      "kl": 0.086181640625,
+      "learning_rate": 8.749333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 938
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.546875,
+      "epoch": 1.5024,
+      "grad_norm": 0.028901370897764077,
+      "kl": 0.08935546875,
+      "learning_rate": 8.748e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 939
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.921875,
+      "epoch": 1.504,
+      "grad_norm": 13.127115307524905,
+      "kl": 0.09814453125,
+      "learning_rate": 8.746666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.703125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.703125,
+      "step": 940
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.375,
+      "epoch": 1.5056,
+      "grad_norm": 7.4631256761149505,
+      "kl": 0.087646484375,
+      "learning_rate": 8.745333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.65625,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.65625,
+      "step": 941
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.984375,
+      "epoch": 1.5072,
+      "grad_norm": 17.379080729708868,
+      "kl": 0.08203125,
+      "learning_rate": 8.743999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.703125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.703125,
+      "step": 942
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.171875,
+      "epoch": 1.5088,
+      "grad_norm": 0.006379913881629091,
+      "kl": 0.091552734375,
+      "learning_rate": 8.742666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 943
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.453125,
+      "epoch": 1.5104,
+      "grad_norm": 27.06689781300175,
+      "kl": 0.08642578125,
+      "learning_rate": 8.741333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.59375,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.59375,
+      "step": 944
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.65625,
+      "epoch": 1.512,
+      "grad_norm": 0.00806817606433293,
+      "kl": 0.0888671875,
+      "learning_rate": 8.739999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 945
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.828125,
+      "epoch": 1.5135999999999998,
+      "grad_norm": 0.00970795761381199,
+      "kl": 0.088623046875,
+      "learning_rate": 8.738666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 946
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.953125,
+      "epoch": 1.5152,
+      "grad_norm": 0.01462760586692234,
+      "kl": 0.09521484375,
+      "learning_rate": 8.737333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 947
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.875,
+      "epoch": 1.5168,
+      "grad_norm": 0.0071018943424659385,
+      "kl": 0.091552734375,
+      "learning_rate": 8.736e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 948
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.109375,
+      "epoch": 1.5184,
+      "grad_norm": 50.47560496590217,
+      "kl": 0.079833984375,
+      "learning_rate": 8.734666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 949
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.09375,
+      "epoch": 1.52,
+      "grad_norm": 0.006971377749329209,
+      "kl": 0.093994140625,
+      "learning_rate": 8.733333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 950
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.640625,
+      "epoch": 1.5215999999999998,
+      "grad_norm": 0.007398395629622126,
+      "kl": 0.087890625,
+      "learning_rate": 8.732e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 951
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.96875,
+      "epoch": 1.5232,
+      "grad_norm": 15.060950804949314,
+      "kl": 0.088134765625,
+      "learning_rate": 8.730666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.359375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.359375,
+      "step": 952
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.375,
+      "epoch": 1.5248,
+      "grad_norm": 0.009020005925351635,
+      "kl": 0.0859375,
+      "learning_rate": 8.729333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 953
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.140625,
+      "epoch": 1.5264,
+      "grad_norm": 0.0059455856703348456,
+      "kl": 0.083984375,
+      "learning_rate": 8.728e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 954
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.953125,
+      "epoch": 1.528,
+      "grad_norm": 0.010859501230833334,
+      "kl": 0.087890625,
+      "learning_rate": 8.726666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.25,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.25,
+      "step": 955
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.171875,
+      "epoch": 1.5295999999999998,
+      "grad_norm": 0.005907535661575243,
+      "kl": 0.086669921875,
+      "learning_rate": 8.725333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 956
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.4375,
+      "epoch": 1.5312000000000001,
+      "grad_norm": 0.006492120152810119,
+      "kl": 0.09033203125,
+      "learning_rate": 8.723999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 957
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.125,
+      "epoch": 1.5328,
+      "grad_norm": 0.0050972949813701636,
+      "kl": 0.079345703125,
+      "learning_rate": 8.722666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 958
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.71875,
+      "epoch": 1.5344,
+      "grad_norm": 0.011722319954031694,
+      "kl": 0.087646484375,
+      "learning_rate": 8.721333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 959
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.796875,
+      "epoch": 1.536,
+      "grad_norm": 0.008275692525796855,
+      "kl": 0.086181640625,
+      "learning_rate": 8.72e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 960
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.65625,
+      "epoch": 1.5375999999999999,
+      "grad_norm": 0.005752396924897746,
+      "kl": 0.09521484375,
+      "learning_rate": 8.718666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 961
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.375,
+      "epoch": 1.5392000000000001,
+      "grad_norm": 0.01392199902367659,
+      "kl": 0.08056640625,
+      "learning_rate": 8.717333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 962
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.09375,
+      "epoch": 1.5408,
+      "grad_norm": 0.027878112242796203,
+      "kl": 0.082763671875,
+      "learning_rate": 8.716e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 963
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.390625,
+      "epoch": 1.5424,
+      "grad_norm": 0.010013725675742366,
+      "kl": 0.084716796875,
+      "learning_rate": 8.714666666666665e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 964
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.234375,
+      "epoch": 1.544,
+      "grad_norm": 0.005477848603364648,
+      "kl": 0.07080078125,
+      "learning_rate": 8.713333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 965
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.234375,
+      "epoch": 1.5455999999999999,
+      "grad_norm": 0.008690292721440049,
+      "kl": 0.09033203125,
+      "learning_rate": 8.711999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 966
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 143.03125,
+      "epoch": 1.5472000000000001,
+      "grad_norm": 0.015506180014386448,
+      "kl": 0.087890625,
+      "learning_rate": 8.710666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 967
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.96875,
+      "epoch": 1.5488,
+      "grad_norm": 0.011466698316203774,
+      "kl": 0.080322265625,
+      "learning_rate": 8.709333333333333e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 968
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.65625,
+      "epoch": 1.5504,
+      "grad_norm": 0.014586503953374831,
+      "kl": 0.09716796875,
+      "learning_rate": 8.708e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 969
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.609375,
+      "epoch": 1.552,
+      "grad_norm": 0.022503182021344575,
+      "kl": 0.088134765625,
+      "learning_rate": 8.706666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 970
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.515625,
+      "epoch": 1.5535999999999999,
+      "grad_norm": 0.00660424973228776,
+      "kl": 0.087646484375,
+      "learning_rate": 8.705333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 971
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.046875,
+      "epoch": 1.5552000000000001,
+      "grad_norm": 4.703555908772052,
+      "kl": 0.080078125,
+      "learning_rate": 8.704e-07,
+      "loss": 0.0001,
+      "reward": 1.9375,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.9375,
+      "step": 972
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.21875,
+      "epoch": 1.5568,
+      "grad_norm": 0.009891577111942853,
+      "kl": 0.083984375,
+      "learning_rate": 8.702666666666665e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 973
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 143.03125,
+      "epoch": 1.5584,
+      "grad_norm": 0.009001156587752185,
+      "kl": 0.0830078125,
+      "learning_rate": 8.701333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 974
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 142.296875,
+      "epoch": 1.56,
+      "grad_norm": 1.139727720641073,
+      "kl": 0.08056640625,
+      "learning_rate": 8.699999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.96875,
+      "reward_std": 0.0883883461356163,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.984375,
+      "step": 975
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.921875,
+      "epoch": 1.5615999999999999,
+      "grad_norm": 0.008400046522377129,
+      "kl": 0.0810546875,
+      "learning_rate": 8.698666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 976
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.5625,
+      "epoch": 1.5632000000000001,
+      "grad_norm": 0.011152748855425916,
+      "kl": 0.083740234375,
+      "learning_rate": 8.697333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 977
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.453125,
+      "epoch": 1.5648,
+      "grad_norm": 75.18962780953794,
+      "kl": 0.08740234375,
+      "learning_rate": 8.696e-07,
+      "loss": 0.0001,
+      "reward": 1.453125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.453125,
+      "step": 978
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.984375,
+      "epoch": 1.5664,
+      "grad_norm": 0.006007766534505148,
+      "kl": 0.081787109375,
+      "learning_rate": 8.694666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 979
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.546875,
+      "epoch": 1.568,
+      "grad_norm": 0.009956758297203801,
+      "kl": 0.08447265625,
+      "learning_rate": 8.693333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 980
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.984375,
+      "epoch": 1.5695999999999999,
+      "grad_norm": 0.005022167770385414,
+      "kl": 0.081787109375,
+      "learning_rate": 8.692e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 981
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.375,
+      "epoch": 1.5712000000000002,
+      "grad_norm": 0.007809561868786673,
+      "kl": 0.085205078125,
+      "learning_rate": 8.690666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 982
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.84375,
+      "epoch": 1.5728,
+      "grad_norm": 0.006684063747945948,
+      "kl": 0.086181640625,
+      "learning_rate": 8.689333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 983
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.59375,
+      "epoch": 1.5744,
+      "grad_norm": 0.010158024063528436,
+      "kl": 0.088623046875,
+      "learning_rate": 8.687999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.25,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.25,
+      "step": 984
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.8125,
+      "epoch": 1.576,
+      "grad_norm": 19.135658695837535,
+      "kl": 0.0849609375,
+      "learning_rate": 8.686666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.859375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.859375,
+      "step": 985
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.3125,
+      "epoch": 1.5776,
+      "grad_norm": 0.007738787938547713,
+      "kl": 0.094970703125,
+      "learning_rate": 8.685333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 986
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.171875,
+      "epoch": 1.5792000000000002,
+      "grad_norm": 0.0056518681202991335,
+      "kl": 0.0810546875,
+      "learning_rate": 8.683999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 987
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.015625,
+      "epoch": 1.5808,
+      "grad_norm": 2.1647978227739313,
+      "kl": 0.091796875,
+      "learning_rate": 8.682666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.859375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.859375,
+      "step": 988
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.859375,
+      "epoch": 1.5824,
+      "grad_norm": 0.010405723276924878,
+      "kl": 0.088134765625,
+      "learning_rate": 8.681333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 989
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.109375,
+      "epoch": 1.584,
+      "grad_norm": 0.010571594072028137,
+      "kl": 0.091796875,
+      "learning_rate": 8.68e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 990
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.625,
+      "epoch": 1.5856,
+      "grad_norm": 0.013255290522795072,
+      "kl": 0.082275390625,
+      "learning_rate": 8.678666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 991
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.6875,
+      "epoch": 1.5872000000000002,
+      "grad_norm": 0.03326702974433086,
+      "kl": 0.08544921875,
+      "learning_rate": 8.677333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 992
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.40625,
+      "epoch": 1.5888,
+      "grad_norm": 0.009717836816668305,
+      "kl": 0.106689453125,
+      "learning_rate": 8.676e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 993
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.625,
+      "epoch": 1.5904,
+      "grad_norm": 0.011757019911465584,
+      "kl": 0.0888671875,
+      "learning_rate": 8.674666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 994
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.703125,
+      "epoch": 1.592,
+      "grad_norm": 27.754218223963505,
+      "kl": 0.08837890625,
+      "learning_rate": 8.673333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.546875,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.546875,
+      "step": 995
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.828125,
+      "epoch": 1.5936,
+      "grad_norm": 0.006498290524168493,
+      "kl": 0.087890625,
+      "learning_rate": 8.671999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 996
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.453125,
+      "epoch": 1.5952,
+      "grad_norm": 0.017994234886183794,
+      "kl": 0.103271484375,
+      "learning_rate": 8.670666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 997
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.390625,
+      "epoch": 1.5968,
+      "grad_norm": 0.07428195012518764,
+      "kl": 0.10107421875,
+      "learning_rate": 8.669333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 998
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.0625,
+      "epoch": 1.5984,
+      "grad_norm": 0.006799511671953883,
+      "kl": 0.099365234375,
+      "learning_rate": 8.668e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 999
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.546875,
+      "epoch": 1.6,
+      "grad_norm": 0.008129562489011982,
+      "kl": 0.08935546875,
+      "learning_rate": 8.666666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1000
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.984375,
+      "epoch": 1.6016,
+      "grad_norm": 0.022295611142657542,
+      "kl": 0.095947265625,
+      "learning_rate": 8.665333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1001
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.671875,
+      "epoch": 1.6032,
+      "grad_norm": 0.005734934975290421,
+      "kl": 0.0888671875,
+      "learning_rate": 8.663999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1002
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.46875,
+      "epoch": 1.6048,
+      "grad_norm": 9.20775485168076,
+      "kl": 0.087158203125,
+      "learning_rate": 8.662666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.4375,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.4375,
+      "step": 1003
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.453125,
+      "epoch": 1.6064,
+      "grad_norm": 3.5064079666209067,
+      "kl": 0.1005859375,
+      "learning_rate": 8.661333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.859375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.859375,
+      "step": 1004
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.234375,
+      "epoch": 1.608,
+      "grad_norm": 4.956366510971427,
+      "kl": 0.089599609375,
+      "learning_rate": 8.659999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.640625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.640625,
+      "step": 1005
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.84375,
+      "epoch": 1.6096,
+      "grad_norm": 8.315846295108388,
+      "kl": 0.084716796875,
+      "learning_rate": 8.658666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.640625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.640625,
+      "step": 1006
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.03125,
+      "epoch": 1.6112,
+      "grad_norm": 0.006047585064245432,
+      "kl": 0.09033203125,
+      "learning_rate": 8.657333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1007
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.953125,
+      "epoch": 1.6128,
+      "grad_norm": 0.03533382486531617,
+      "kl": 0.096435546875,
+      "learning_rate": 8.656e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1008
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.703125,
+      "epoch": 1.6143999999999998,
+      "grad_norm": 0.007489079312660206,
+      "kl": 0.08251953125,
+      "learning_rate": 8.654666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1009
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.96875,
+      "epoch": 1.616,
+      "grad_norm": 13.401766785440273,
+      "kl": 0.09033203125,
+      "learning_rate": 8.653333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.546875,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.546875,
+      "step": 1010
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.359375,
+      "epoch": 1.6176,
+      "grad_norm": 0.006928522409609723,
+      "kl": 0.084228515625,
+      "learning_rate": 8.651999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1011
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.875,
+      "epoch": 1.6192,
+      "grad_norm": 0.005562269255987464,
+      "kl": 0.083251953125,
+      "learning_rate": 8.650666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1012
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.84375,
+      "epoch": 1.6208,
+      "grad_norm": 0.0042650687838736625,
+      "kl": 0.071533203125,
+      "learning_rate": 8.649333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1013
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.375,
+      "epoch": 1.6223999999999998,
+      "grad_norm": 0.006744490670045065,
+      "kl": 0.083251953125,
+      "learning_rate": 8.648e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1014
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.359375,
+      "epoch": 1.624,
+      "grad_norm": 14.537998029484775,
+      "kl": 0.08251953125,
+      "learning_rate": 8.646666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.609375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.609375,
+      "step": 1015
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.125,
+      "epoch": 1.6256,
+      "grad_norm": 0.007843793650475555,
+      "kl": 0.08447265625,
+      "learning_rate": 8.645333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1016
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.609375,
+      "epoch": 1.6272,
+      "grad_norm": 9.57209356418108,
+      "kl": 0.078857421875,
+      "learning_rate": 8.643999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.65625,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.65625,
+      "step": 1017
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.015625,
+      "epoch": 1.6288,
+      "grad_norm": 0.01101811128236686,
+      "kl": 0.07958984375,
+      "learning_rate": 8.642666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1018
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.78125,
+      "epoch": 1.6303999999999998,
+      "grad_norm": 8.255568514419556,
+      "kl": 0.07861328125,
+      "learning_rate": 8.641333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.65625,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.65625,
+      "step": 1019
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.71875,
+      "epoch": 1.6320000000000001,
+      "grad_norm": 0.006286173576750012,
+      "kl": 0.093017578125,
+      "learning_rate": 8.639999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1020
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.921875,
+      "epoch": 1.6336,
+      "grad_norm": 0.006522709671986863,
+      "kl": 0.082275390625,
+      "learning_rate": 8.638666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1021
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.28125,
+      "epoch": 1.6352,
+      "grad_norm": 5.9492462264427814,
+      "kl": 0.082763671875,
+      "learning_rate": 8.637333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.640625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.640625,
+      "step": 1022
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.25,
+      "epoch": 1.6368,
+      "grad_norm": 0.005654706846772655,
+      "kl": 0.07666015625,
+      "learning_rate": 8.636e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 1023
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.015625,
+      "epoch": 1.6383999999999999,
+      "grad_norm": 3.9100321603939587,
+      "kl": 0.08544921875,
+      "learning_rate": 8.634666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.46875,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.46875,
+      "step": 1024
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.25,
+      "epoch": 1.6400000000000001,
+      "grad_norm": 0.015774038132855658,
+      "kl": 0.083984375,
+      "learning_rate": 8.633333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1025
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.34375,
+      "epoch": 1.6416,
+      "grad_norm": 0.027437305272229248,
+      "kl": 0.084228515625,
+      "learning_rate": 8.632e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1026
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.453125,
+      "epoch": 1.6432,
+      "grad_norm": 0.004960609183657568,
+      "kl": 0.08251953125,
+      "learning_rate": 8.630666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1027
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.53125,
+      "epoch": 1.6448,
+      "grad_norm": 0.0066999600511313765,
+      "kl": 0.08154296875,
+      "learning_rate": 8.629333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1028
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.078125,
+      "epoch": 1.6463999999999999,
+      "grad_norm": 0.0055132452476678645,
+      "kl": 0.079345703125,
+      "learning_rate": 8.628e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1029
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.5625,
+      "epoch": 1.6480000000000001,
+      "grad_norm": 0.006575924299944727,
+      "kl": 0.078857421875,
+      "learning_rate": 8.626666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1030
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.0,
+      "epoch": 1.6496,
+      "grad_norm": 0.006343425906647738,
+      "kl": 0.081298828125,
+      "learning_rate": 8.625333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1031
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.578125,
+      "epoch": 1.6512,
+      "grad_norm": 0.005710985426011118,
+      "kl": 0.084716796875,
+      "learning_rate": 8.624e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1032
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.625,
+      "epoch": 1.6528,
+      "grad_norm": 0.018161924622432005,
+      "kl": 0.08740234375,
+      "learning_rate": 8.622666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.25,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.25,
+      "step": 1033
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.421875,
+      "epoch": 1.6543999999999999,
+      "grad_norm": 0.009408692731594554,
+      "kl": 0.081298828125,
+      "learning_rate": 8.621333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1034
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.0,
+      "epoch": 1.6560000000000001,
+      "grad_norm": 0.006368015844430695,
+      "kl": 0.087890625,
+      "learning_rate": 8.62e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1035
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.65625,
+      "epoch": 1.6576,
+      "grad_norm": 0.005129059074474503,
+      "kl": 0.088623046875,
+      "learning_rate": 8.618666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1036
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.40625,
+      "epoch": 1.6592,
+      "grad_norm": 0.008063968569963916,
+      "kl": 0.079345703125,
+      "learning_rate": 8.617333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1037
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.21875,
+      "epoch": 1.6608,
+      "grad_norm": 0.005288412221150649,
+      "kl": 0.07763671875,
+      "learning_rate": 8.616e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1038
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.265625,
+      "epoch": 1.6623999999999999,
+      "grad_norm": 0.006827523154127113,
+      "kl": 0.083740234375,
+      "learning_rate": 8.614666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1039
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 142.0,
+      "epoch": 1.6640000000000001,
+      "grad_norm": 0.006748442428079407,
+      "kl": 0.082275390625,
+      "learning_rate": 8.613333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1040
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.296875,
+      "epoch": 1.6656,
+      "grad_norm": 1.498668906450298,
+      "kl": 0.08349609375,
+      "learning_rate": 8.611999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.359375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.375,
+      "step": 1041
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.796875,
+      "epoch": 1.6672,
+      "grad_norm": 0.0058959176902399725,
+      "kl": 0.077880859375,
+      "learning_rate": 8.610666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1042
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.234375,
+      "epoch": 1.6688,
+      "grad_norm": 0.00707276555928845,
+      "kl": 0.096435546875,
+      "learning_rate": 8.609333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1043
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.15625,
+      "epoch": 1.6703999999999999,
+      "grad_norm": 0.018401769341020533,
+      "kl": 0.083740234375,
+      "learning_rate": 8.608e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1044
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.703125,
+      "epoch": 1.6720000000000002,
+      "grad_norm": 22.17083838620716,
+      "kl": 0.079345703125,
+      "learning_rate": 8.606666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.46875,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.46875,
+      "step": 1045
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.140625,
+      "epoch": 1.6736,
+      "grad_norm": 8.132243880164415,
+      "kl": 0.081787109375,
+      "learning_rate": 8.605333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.765625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.765625,
+      "step": 1046
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.125,
+      "epoch": 1.6752,
+      "grad_norm": 0.005124135628389918,
+      "kl": 0.081787109375,
+      "learning_rate": 8.604000000000001e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1047
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.34375,
+      "epoch": 1.6768,
+      "grad_norm": 0.004941813077544285,
+      "kl": 0.085205078125,
+      "learning_rate": 8.602666666666665e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1048
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 142.78125,
+      "epoch": 1.6784,
+      "grad_norm": 0.014545227265655368,
+      "kl": 0.08056640625,
+      "learning_rate": 8.601333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1049
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 141.9375,
+      "epoch": 1.6800000000000002,
+      "grad_norm": 0.008868683462237835,
+      "kl": 0.083251953125,
+      "learning_rate": 8.599999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1050
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.21875,
+      "epoch": 1.6816,
+      "grad_norm": 0.006337758092775204,
+      "kl": 0.0810546875,
+      "learning_rate": 8.598666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1051
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.125,
+      "epoch": 1.6832,
+      "grad_norm": 0.005239005797705131,
+      "kl": 0.087158203125,
+      "learning_rate": 8.597333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1052
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.0,
+      "epoch": 1.6848,
+      "grad_norm": 0.004716271678969138,
+      "kl": 0.0849609375,
+      "learning_rate": 8.596e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1053
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 143.796875,
+      "epoch": 1.6864,
+      "grad_norm": 0.006899208614437051,
+      "kl": 0.091552734375,
+      "learning_rate": 8.594666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.25,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.25,
+      "step": 1054
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.625,
+      "epoch": 1.688,
+      "grad_norm": 0.00648684819688244,
+      "kl": 0.087890625,
+      "learning_rate": 8.593333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1055
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.453125,
+      "epoch": 1.6896,
+      "grad_norm": 0.005306994444757418,
+      "kl": 0.079345703125,
+      "learning_rate": 8.592e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1056
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.15625,
+      "epoch": 1.6912,
+      "grad_norm": 0.004658265461859795,
+      "kl": 0.08056640625,
+      "learning_rate": 8.590666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1057
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.0625,
+      "epoch": 1.6928,
+      "grad_norm": 0.011214256825753699,
+      "kl": 0.0908203125,
+      "learning_rate": 8.589333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1058
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 144.703125,
+      "epoch": 1.6944,
+      "grad_norm": 0.005235097228919457,
+      "kl": 0.084716796875,
+      "learning_rate": 8.587999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1059
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 148.90625,
+      "epoch": 1.696,
+      "grad_norm": 0.00917954516037095,
+      "kl": 0.081787109375,
+      "learning_rate": 8.586666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1060
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 146.296875,
+      "epoch": 1.6976,
+      "grad_norm": 0.005483575946871765,
+      "kl": 0.092529296875,
+      "learning_rate": 8.585333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1061
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 142.84375,
+      "epoch": 1.6992,
+      "grad_norm": 0.008079898786462883,
+      "kl": 0.08544921875,
+      "learning_rate": 8.584e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1062
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 145.6875,
+      "epoch": 1.7008,
+      "grad_norm": 0.012829626813950428,
+      "kl": 0.0888671875,
+      "learning_rate": 8.582666666666666e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 1063
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 145.203125,
+      "epoch": 1.7024,
+      "grad_norm": 0.005819059083367649,
+      "kl": 0.084228515625,
+      "learning_rate": 8.581333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1064
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 152.609375,
+      "epoch": 1.704,
+      "grad_norm": 0.016950320147544183,
+      "kl": 0.08642578125,
+      "learning_rate": 8.58e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 1065
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 145.171875,
+      "epoch": 1.7056,
+      "grad_norm": 0.005839848173835065,
+      "kl": 0.09033203125,
+      "learning_rate": 8.578666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1066
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 143.15625,
+      "epoch": 1.7072,
+      "grad_norm": 0.005459071763789093,
+      "kl": 0.08447265625,
+      "learning_rate": 8.577333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1067
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.390625,
+      "epoch": 1.7088,
+      "grad_norm": 0.004660518960247767,
+      "kl": 0.076416015625,
+      "learning_rate": 8.576e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1068
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 151.375,
+      "epoch": 1.7104,
+      "grad_norm": 1.0737537315237922,
+      "kl": 0.0791015625,
+      "learning_rate": 8.574666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.484375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.5,
+      "step": 1069
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.734375,
+      "epoch": 1.712,
+      "grad_norm": 0.006333813369965715,
+      "kl": 0.074462890625,
+      "learning_rate": 8.573333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1070
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 141.859375,
+      "epoch": 1.7136,
+      "grad_norm": 0.0064163049309609,
+      "kl": 0.078857421875,
+      "learning_rate": 8.571999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1071
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.515625,
+      "epoch": 1.7151999999999998,
+      "grad_norm": 0.007787617114439567,
+      "kl": 0.080322265625,
+      "learning_rate": 8.570666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1072
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.859375,
+      "epoch": 1.7168,
+      "grad_norm": 0.007070612517919574,
+      "kl": 0.09228515625,
+      "learning_rate": 8.569333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1073
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.734375,
+      "epoch": 1.7184,
+      "grad_norm": 35.362239374612436,
+      "kl": 0.085205078125,
+      "learning_rate": 8.568e-07,
+      "loss": 0.0001,
+      "reward": 1.890625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.890625,
+      "step": 1074
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.234375,
+      "epoch": 1.72,
+      "grad_norm": 0.005745759208708541,
+      "kl": 0.090087890625,
+      "learning_rate": 8.566666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1075
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.0,
+      "epoch": 1.7216,
+      "grad_norm": 0.0060749191406052954,
+      "kl": 0.089599609375,
+      "learning_rate": 8.565333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1076
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.375,
+      "epoch": 1.7231999999999998,
+      "grad_norm": 24.184745535929732,
+      "kl": 0.099853515625,
+      "learning_rate": 8.564e-07,
+      "loss": 0.0001,
+      "reward": 1.578125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.578125,
+      "step": 1077
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.0625,
+      "epoch": 1.7248,
+      "grad_norm": 16.123650001836165,
+      "kl": 0.0927734375,
+      "learning_rate": 8.562666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.65625,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.65625,
+      "step": 1078
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.171875,
+      "epoch": 1.7264,
+      "grad_norm": 0.0047811434418267495,
+      "kl": 0.08837890625,
+      "learning_rate": 8.561333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1079
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.828125,
+      "epoch": 1.728,
+      "grad_norm": 0.006993544072613898,
+      "kl": 0.0810546875,
+      "learning_rate": 8.559999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1080
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.078125,
+      "epoch": 1.7296,
+      "grad_norm": 0.006161593818652083,
+      "kl": 0.09375,
+      "learning_rate": 8.558666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1081
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.890625,
+      "epoch": 1.7311999999999999,
+      "grad_norm": 24.513290954673142,
+      "kl": 0.088134765625,
+      "learning_rate": 8.557333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.6875,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.6875,
+      "step": 1082
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.609375,
+      "epoch": 1.7328000000000001,
+      "grad_norm": 3.472778088358826,
+      "kl": 0.100830078125,
+      "learning_rate": 8.556e-07,
+      "loss": 0.0001,
+      "reward": 1.78125,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.78125,
+      "step": 1083
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.3125,
+      "epoch": 1.7344,
+      "grad_norm": 0.013433317555071662,
+      "kl": 0.0908203125,
+      "learning_rate": 8.554666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1084
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.0625,
+      "epoch": 1.736,
+      "grad_norm": 0.028570471899779296,
+      "kl": 0.09375,
+      "learning_rate": 8.553333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1085
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.015625,
+      "epoch": 1.7376,
+      "grad_norm": 9.549481625245257,
+      "kl": 0.09423828125,
+      "learning_rate": 8.551999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.65625,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.65625,
+      "step": 1086
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.21875,
+      "epoch": 1.7391999999999999,
+      "grad_norm": 0.005751034194521238,
+      "kl": 0.09765625,
+      "learning_rate": 8.550666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1087
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.859375,
+      "epoch": 1.7408000000000001,
+      "grad_norm": 0.00760476056541643,
+      "kl": 0.108642578125,
+      "learning_rate": 8.549333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1088
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.90625,
+      "epoch": 1.7424,
+      "grad_norm": 0.007420952526176733,
+      "kl": 0.09375,
+      "learning_rate": 8.548e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1089
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.546875,
+      "epoch": 1.744,
+      "grad_norm": 0.008711760482938858,
+      "kl": 0.09619140625,
+      "learning_rate": 8.546666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1090
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.984375,
+      "epoch": 1.7456,
+      "grad_norm": 8.464112226147074,
+      "kl": 0.10009765625,
+      "learning_rate": 8.545333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.515625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.515625,
+      "step": 1091
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.0625,
+      "epoch": 1.7471999999999999,
+      "grad_norm": 0.011955718364139176,
+      "kl": 0.09228515625,
+      "learning_rate": 8.544e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1092
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.859375,
+      "epoch": 1.7488000000000001,
+      "grad_norm": 0.027211412001295208,
+      "kl": 0.10009765625,
+      "learning_rate": 8.542666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1093
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.4375,
+      "epoch": 1.7504,
+      "grad_norm": 0.010820152285324786,
+      "kl": 0.1005859375,
+      "learning_rate": 8.541333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1094
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.78125,
+      "epoch": 1.752,
+      "grad_norm": 0.005561855344876993,
+      "kl": 0.08544921875,
+      "learning_rate": 8.539999999999999e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 1095
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.109375,
+      "epoch": 1.7536,
+      "grad_norm": 0.005032700309747837,
+      "kl": 0.08349609375,
+      "learning_rate": 8.538666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1096
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.765625,
+      "epoch": 1.7551999999999999,
+      "grad_norm": 0.01810934619117422,
+      "kl": 0.08984375,
+      "learning_rate": 8.537333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1097
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.234375,
+      "epoch": 1.7568000000000001,
+      "grad_norm": 1.096613929717187,
+      "kl": 0.08642578125,
+      "learning_rate": 8.536e-07,
+      "loss": 0.0001,
+      "reward": 1.765625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.765625,
+      "step": 1098
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.109375,
+      "epoch": 1.7584,
+      "grad_norm": 0.0066945011943651414,
+      "kl": 0.090576171875,
+      "learning_rate": 8.534666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1099
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.703125,
+      "epoch": 1.76,
+      "grad_norm": 0.010634233183651714,
+      "kl": 0.091796875,
+      "learning_rate": 8.533333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.25,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.25,
+      "step": 1100
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.796875,
+      "epoch": 1.7616,
+      "grad_norm": 0.005781681585207584,
+      "kl": 0.09814453125,
+      "learning_rate": 8.531999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1101
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 144.5625,
+      "epoch": 1.7631999999999999,
+      "grad_norm": 0.00758715641208433,
+      "kl": 0.0830078125,
+      "learning_rate": 8.530666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1102
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 148.453125,
+      "epoch": 1.7648000000000001,
+      "grad_norm": 0.007085288384677996,
+      "kl": 0.091064453125,
+      "learning_rate": 8.529333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1103
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 143.53125,
+      "epoch": 1.7664,
+      "grad_norm": 0.006386737175541498,
+      "kl": 0.094482421875,
+      "learning_rate": 8.528e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1104
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 149.140625,
+      "epoch": 1.768,
+      "grad_norm": 27.41123407275934,
+      "kl": 0.089111328125,
+      "learning_rate": 8.526666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.828125,
+      "reward_std": 0.12255740165710449,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.828125,
+      "step": 1105
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 152.21875,
+      "epoch": 1.7696,
+      "grad_norm": 0.008339576966437494,
+      "kl": 0.1064453125,
+      "learning_rate": 8.525333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1106
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 157.84375,
+      "epoch": 1.7711999999999999,
+      "grad_norm": 0.005351731673028733,
+      "kl": 0.089111328125,
+      "learning_rate": 8.524e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1107
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 153.421875,
+      "epoch": 1.7728000000000002,
+      "grad_norm": 0.922888915303191,
+      "kl": 0.10498046875,
+      "learning_rate": 8.522666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.46875,
+      "reward_std": 0.0883883461356163,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.484375,
+      "step": 1108
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 157.75,
+      "epoch": 1.7744,
+      "grad_norm": 0.005431562675187918,
+      "kl": 0.09716796875,
+      "learning_rate": 8.521333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1109
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 148.40625,
+      "epoch": 1.776,
+      "grad_norm": 31.84191561342175,
+      "kl": 0.107421875,
+      "learning_rate": 8.52e-07,
+      "loss": 0.0001,
+      "reward": 1.671875,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.671875,
+      "step": 1110
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 146.921875,
+      "epoch": 1.7776,
+      "grad_norm": 0.006781684956335393,
+      "kl": 0.105224609375,
+      "learning_rate": 8.518666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1111
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 146.359375,
+      "epoch": 1.7792,
+      "grad_norm": 0.026565873544845974,
+      "kl": 0.10595703125,
+      "learning_rate": 8.517333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1112
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 143.109375,
+      "epoch": 1.7808000000000002,
+      "grad_norm": 0.007528954283658925,
+      "kl": 0.114501953125,
+      "learning_rate": 8.516e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1113
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.859375,
+      "epoch": 1.7824,
+      "grad_norm": 20.043995965166246,
+      "kl": 0.118408203125,
+      "learning_rate": 8.514666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.484375,
+      "reward_std": 0.10205793008208275,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.484375,
+      "step": 1114
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.3125,
+      "epoch": 1.784,
+      "grad_norm": 0.013042571678568666,
+      "kl": 0.116455078125,
+      "learning_rate": 8.513333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1115
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.53125,
+      "epoch": 1.7856,
+      "grad_norm": 0.0070265568932148794,
+      "kl": 0.12158203125,
+      "learning_rate": 8.511999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1116
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.09375,
+      "epoch": 1.7872,
+      "grad_norm": 0.006491727614208869,
+      "kl": 0.12109375,
+      "learning_rate": 8.510666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1117
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.71875,
+      "epoch": 1.7888,
+      "grad_norm": 0.011680935901247119,
+      "kl": 0.12548828125,
+      "learning_rate": 8.509333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1118
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.046875,
+      "epoch": 1.7904,
+      "grad_norm": 0.023036409745528046,
+      "kl": 0.123779296875,
+      "learning_rate": 8.508e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1119
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.828125,
+      "epoch": 1.792,
+      "grad_norm": 0.04093647152090056,
+      "kl": 0.122314453125,
+      "learning_rate": 8.506666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1120
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.0625,
+      "epoch": 1.7936,
+      "grad_norm": 0.009976710675448057,
+      "kl": 0.132568359375,
+      "learning_rate": 8.505333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1121
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.96875,
+      "epoch": 1.7952,
+      "grad_norm": 0.006827521311702167,
+      "kl": 0.12109375,
+      "learning_rate": 8.504e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1122
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.40625,
+      "epoch": 1.7968,
+      "grad_norm": 0.007793729465120075,
+      "kl": 0.12646484375,
+      "learning_rate": 8.502666666666665e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1123
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.671875,
+      "epoch": 1.7984,
+      "grad_norm": 0.022662670183971164,
+      "kl": 0.1279296875,
+      "learning_rate": 8.501333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1124
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.390625,
+      "epoch": 1.8,
+      "grad_norm": 0.008825017997623653,
+      "kl": 0.125,
+      "learning_rate": 8.499999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1125
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.359375,
+      "epoch": 1.8016,
+      "grad_norm": 0.007897337289020583,
+      "kl": 0.12158203125,
+      "learning_rate": 8.498666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1126
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.984375,
+      "epoch": 1.8032,
+      "grad_norm": 72.10891969861689,
+      "kl": 0.130126953125,
+      "learning_rate": 8.497333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.703125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.703125,
+      "step": 1127
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.234375,
+      "epoch": 1.8048,
+      "grad_norm": 3.0443715690683533,
+      "kl": 0.129638671875,
+      "learning_rate": 8.496e-07,
+      "loss": 0.0001,
+      "reward": 1.40625,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.40625,
+      "step": 1128
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.3125,
+      "epoch": 1.8064,
+      "grad_norm": 0.011221013089650159,
+      "kl": 0.12646484375,
+      "learning_rate": 8.494666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1129
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.46875,
+      "epoch": 1.808,
+      "grad_norm": 0.011669381556000322,
+      "kl": 0.1240234375,
+      "learning_rate": 8.493333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1130
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.015625,
+      "epoch": 1.8096,
+      "grad_norm": 1.4650588397840483,
+      "kl": 0.132568359375,
+      "learning_rate": 8.492e-07,
+      "loss": 0.0001,
+      "reward": 1.515625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.515625,
+      "step": 1131
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.09375,
+      "epoch": 1.8112,
+      "grad_norm": 0.01049652184545184,
+      "kl": 0.13427734375,
+      "learning_rate": 8.490666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1132
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.703125,
+      "epoch": 1.8128,
+      "grad_norm": 7.688595554831467,
+      "kl": 0.13330078125,
+      "learning_rate": 8.489333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.28125,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.28125,
+      "step": 1133
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.3125,
+      "epoch": 1.8144,
+      "grad_norm": 1.539694276442812,
+      "kl": 0.140625,
+      "learning_rate": 8.487999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.5625,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5625,
+      "step": 1134
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.390625,
+      "epoch": 1.8159999999999998,
+      "grad_norm": 0.008819744854011725,
+      "kl": 0.15478515625,
+      "learning_rate": 8.486666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1135
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.75,
+      "epoch": 1.8176,
+      "grad_norm": 0.009105708211135691,
+      "kl": 0.140625,
+      "learning_rate": 8.485333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1136
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.296875,
+      "epoch": 1.8192,
+      "grad_norm": 4.253788608208264,
+      "kl": 0.1494140625,
+      "learning_rate": 8.484e-07,
+      "loss": 0.0001,
+      "reward": 1.65625,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.65625,
+      "step": 1137
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.609375,
+      "epoch": 1.8208,
+      "grad_norm": 0.009122272522091033,
+      "kl": 0.14013671875,
+      "learning_rate": 8.482666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1138
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.5,
+      "epoch": 1.8224,
+      "grad_norm": 0.007666854728475739,
+      "kl": 0.14306640625,
+      "learning_rate": 8.481333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1139
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.84375,
+      "epoch": 1.8239999999999998,
+      "grad_norm": 0.010271849168648433,
+      "kl": 0.1611328125,
+      "learning_rate": 8.48e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1140
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.46875,
+      "epoch": 1.8256000000000001,
+      "grad_norm": 0.010091320208708753,
+      "kl": 0.15966796875,
+      "learning_rate": 8.478666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1141
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.03125,
+      "epoch": 1.8272,
+      "grad_norm": 0.0076304601759693445,
+      "kl": 0.1591796875,
+      "learning_rate": 8.477333333333332e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1142
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.078125,
+      "epoch": 1.8288,
+      "grad_norm": 2.781606749799046,
+      "kl": 0.1669921875,
+      "learning_rate": 8.475999999999999e-07,
+      "loss": 0.0002,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.75,
+      "step": 1143
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.734375,
+      "epoch": 1.8304,
+      "grad_norm": 0.010339027865346441,
+      "kl": 0.16259765625,
+      "learning_rate": 8.474666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1144
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.28125,
+      "epoch": 1.8319999999999999,
+      "grad_norm": 6.8755426889272115,
+      "kl": 0.162109375,
+      "learning_rate": 8.473333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 1145
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.625,
+      "epoch": 1.8336000000000001,
+      "grad_norm": 0.008497072726180596,
+      "kl": 0.1787109375,
+      "learning_rate": 8.471999999999999e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1146
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.015625,
+      "epoch": 1.8352,
+      "grad_norm": 0.009296288567504954,
+      "kl": 0.16552734375,
+      "learning_rate": 8.470666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1147
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.875,
+      "epoch": 1.8368,
+      "grad_norm": 0.010327913394295063,
+      "kl": 0.171875,
+      "learning_rate": 8.469333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1148
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.609375,
+      "epoch": 1.8384,
+      "grad_norm": 5.574360442131751,
+      "kl": 0.1826171875,
+      "learning_rate": 8.468e-07,
+      "loss": 0.0002,
+      "reward": 1.71875,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.71875,
+      "step": 1149
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.59375,
+      "epoch": 1.8399999999999999,
+      "grad_norm": 0.008584606497985467,
+      "kl": 0.16796875,
+      "learning_rate": 8.466666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1150
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.5625,
+      "epoch": 1.8416000000000001,
+      "grad_norm": 7.5152887103792585,
+      "kl": 0.1884765625,
+      "learning_rate": 8.465333333333334e-07,
+      "loss": 0.0002,
+      "reward": 1.828125,
+      "reward_std": 0.12255740165710449,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.828125,
+      "step": 1151
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.421875,
+      "epoch": 1.8432,
+      "grad_norm": 0.009243739169919662,
+      "kl": 0.16943359375,
+      "learning_rate": 8.464e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1152
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.765625,
+      "epoch": 1.8448,
+      "grad_norm": 0.07699241593237208,
+      "kl": 0.17431640625,
+      "learning_rate": 8.462666666666665e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1153
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.453125,
+      "epoch": 1.8464,
+      "grad_norm": 8.971264811434722,
+      "kl": 0.21142578125,
+      "learning_rate": 8.461333333333332e-07,
+      "loss": 0.0002,
+      "reward": 1.40625,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.40625,
+      "step": 1154
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.46875,
+      "epoch": 1.8479999999999999,
+      "grad_norm": 0.008533201966930035,
+      "kl": 0.189453125,
+      "learning_rate": 8.459999999999999e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1155
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.15625,
+      "epoch": 1.8496000000000001,
+      "grad_norm": 1.2800397460127397,
+      "kl": 0.19921875,
+      "learning_rate": 8.458666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.59375,
+      "reward_std": 0.0883883461356163,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.609375,
+      "step": 1156
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.421875,
+      "epoch": 1.8512,
+      "grad_norm": 0.008752964361892306,
+      "kl": 0.17138671875,
+      "learning_rate": 8.457333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1157
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.296875,
+      "epoch": 1.8528,
+      "grad_norm": 0.009688770450128675,
+      "kl": 0.1865234375,
+      "learning_rate": 8.456e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1158
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.921875,
+      "epoch": 1.8544,
+      "grad_norm": 0.011978342835581633,
+      "kl": 0.19189453125,
+      "learning_rate": 8.454666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1159
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.234375,
+      "epoch": 1.8559999999999999,
+      "grad_norm": 6.613309680768773,
+      "kl": 0.18505859375,
+      "learning_rate": 8.453333333333334e-07,
+      "loss": 0.0002,
+      "reward": 1.515625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.515625,
+      "step": 1160
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.078125,
+      "epoch": 1.8576000000000001,
+      "grad_norm": 0.017760558235453163,
+      "kl": 0.18505859375,
+      "learning_rate": 8.451999999999999e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1161
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.5,
+      "epoch": 1.8592,
+      "grad_norm": 3.5153387487966232,
+      "kl": 0.1669921875,
+      "learning_rate": 8.450666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.546875,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.546875,
+      "step": 1162
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.59375,
+      "epoch": 1.8608,
+      "grad_norm": 0.01067302156291687,
+      "kl": 0.18310546875,
+      "learning_rate": 8.449333333333332e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1163
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.390625,
+      "epoch": 1.8624,
+      "grad_norm": 19.24036229216912,
+      "kl": 0.17431640625,
+      "learning_rate": 8.447999999999999e-07,
+      "loss": 0.0002,
+      "reward": 1.703125,
+      "reward_std": 0.11100948229432106,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.703125,
+      "step": 1164
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.75,
+      "epoch": 1.8639999999999999,
+      "grad_norm": 0.011578359470359244,
+      "kl": 0.1669921875,
+      "learning_rate": 8.446666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1165
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.328125,
+      "epoch": 1.8656000000000001,
+      "grad_norm": 1.1952010820481926,
+      "kl": 0.1708984375,
+      "learning_rate": 8.445333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.828125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.828125,
+      "step": 1166
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.734375,
+      "epoch": 1.8672,
+      "grad_norm": 1.1383602380748359,
+      "kl": 0.18115234375,
+      "learning_rate": 8.444e-07,
+      "loss": 0.0002,
+      "reward": 1.984375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 1.0,
+      "step": 1167
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.125,
+      "epoch": 1.8688,
+      "grad_norm": 1.1473547160744337,
+      "kl": 0.16748046875,
+      "learning_rate": 8.442666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 1168
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.75,
+      "epoch": 1.8704,
+      "grad_norm": 1.3310012027385503,
+      "kl": 0.1611328125,
+      "learning_rate": 8.441333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.609375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.609375,
+      "step": 1169
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.375,
+      "epoch": 1.8719999999999999,
+      "grad_norm": 0.012450716077417285,
+      "kl": 0.1611328125,
+      "learning_rate": 8.439999999999999e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1170
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.421875,
+      "epoch": 1.8736000000000002,
+      "grad_norm": 1.7387092463139695,
+      "kl": 0.1806640625,
+      "learning_rate": 8.438666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.765625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.765625,
+      "step": 1171
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.984375,
+      "epoch": 1.8752,
+      "grad_norm": 0.009733648796199354,
+      "kl": 0.16064453125,
+      "learning_rate": 8.437333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1172
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.8125,
+      "epoch": 1.8768,
+      "grad_norm": 0.014636846952858917,
+      "kl": 0.1728515625,
+      "learning_rate": 8.436e-07,
+      "loss": 0.0002,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1173
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.515625,
+      "epoch": 1.8784,
+      "grad_norm": 0.011804918944414163,
+      "kl": 0.17041015625,
+      "learning_rate": 8.434666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1174
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.890625,
+      "epoch": 1.88,
+      "grad_norm": 4.574095402346987,
+      "kl": 0.16015625,
+      "learning_rate": 8.433333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.703125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.703125,
+      "step": 1175
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.0,
+      "epoch": 1.8816000000000002,
+      "grad_norm": 0.031841608868668374,
+      "kl": 0.185546875,
+      "learning_rate": 8.431999999999999e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1176
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.640625,
+      "epoch": 1.8832,
+      "grad_norm": 2.080770150135623,
+      "kl": 0.15966796875,
+      "learning_rate": 8.430666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.640625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.640625,
+      "step": 1177
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 100.546875,
+      "epoch": 1.8848,
+      "grad_norm": 0.020935673630403336,
+      "kl": 0.18603515625,
+      "learning_rate": 8.429333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1178
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 94.15625,
+      "epoch": 1.8864,
+      "grad_norm": 0.012420235438464369,
+      "kl": 0.18408203125,
+      "learning_rate": 8.428e-07,
+      "loss": 0.0002,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1179
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 97.8125,
+      "epoch": 1.888,
+      "grad_norm": 1.3948820440767284,
+      "kl": 0.1904296875,
+      "learning_rate": 8.426666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.609375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.625,
+      "step": 1180
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 95.03125,
+      "epoch": 1.8896,
+      "grad_norm": 22.37422919383646,
+      "kl": 0.18994140625,
+      "learning_rate": 8.425333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.71875,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.71875,
+      "step": 1181
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 96.3125,
+      "epoch": 1.8912,
+      "grad_norm": 0.009094717825622498,
+      "kl": 0.20166015625,
+      "learning_rate": 8.424e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1182
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 95.15625,
+      "epoch": 1.8928,
+      "grad_norm": 14.131119074307783,
+      "kl": 0.1923828125,
+      "learning_rate": 8.422666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.390625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.390625,
+      "step": 1183
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 98.5625,
+      "epoch": 1.8944,
+      "grad_norm": 20.549855793368785,
+      "kl": 0.18212890625,
+      "learning_rate": 8.421333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.875,
+      "reward_std": 0.0883883461356163,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1184
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 96.546875,
+      "epoch": 1.896,
+      "grad_norm": 35.690471183879055,
+      "kl": 0.1982421875,
+      "learning_rate": 8.419999999999999e-07,
+      "loss": 0.0002,
+      "reward": 1.84375,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.84375,
+      "step": 1185
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 99.75,
+      "epoch": 1.8976,
+      "grad_norm": 0.0205420050475427,
+      "kl": 0.201171875,
+      "learning_rate": 8.418666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1186
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 99.421875,
+      "epoch": 1.8992,
+      "grad_norm": 0.010668130964616409,
+      "kl": 0.203125,
+      "learning_rate": 8.417333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1187
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 98.015625,
+      "epoch": 1.9008,
+      "grad_norm": 135.1943545767904,
+      "kl": 0.23291015625,
+      "learning_rate": 8.416e-07,
+      "loss": 0.0002,
+      "reward": 1.640625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.640625,
+      "step": 1188
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 100.71875,
+      "epoch": 1.9024,
+      "grad_norm": 0.9557067997144256,
+      "kl": 0.20654296875,
+      "learning_rate": 8.414666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.609375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.609375,
+      "step": 1189
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 92.171875,
+      "epoch": 1.904,
+      "grad_norm": 0.015312773241869158,
+      "kl": 0.220703125,
+      "learning_rate": 8.413333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1190
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 88.71875,
+      "epoch": 1.9056,
+      "grad_norm": 0.014695682096271947,
+      "kl": 0.220703125,
+      "learning_rate": 8.411999999999999e-07,
+      "loss": 0.0002,
+      "reward": 1.125,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.125,
+      "step": 1191
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 95.71875,
+      "epoch": 1.9072,
+      "grad_norm": 0.03044910331336985,
+      "kl": 0.234375,
+      "learning_rate": 8.410666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1192
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 88.90625,
+      "epoch": 1.9088,
+      "grad_norm": 0.0141364930684059,
+      "kl": 0.26416015625,
+      "learning_rate": 8.409333333333333e-07,
+      "loss": 0.0003,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1193
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 85.96875,
+      "epoch": 1.9104,
+      "grad_norm": 1.3866534095404421,
+      "kl": 0.2353515625,
+      "learning_rate": 8.408e-07,
+      "loss": 0.0002,
+      "reward": 1.515625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.515625,
+      "step": 1194
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 87.78125,
+      "epoch": 1.912,
+      "grad_norm": 2.819470933346172,
+      "kl": 0.2392578125,
+      "learning_rate": 8.406666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 1195
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 92.984375,
+      "epoch": 1.9136,
+      "grad_norm": 0.024682775339711326,
+      "kl": 0.234375,
+      "learning_rate": 8.405333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1196
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 87.71875,
+      "epoch": 1.9152,
+      "grad_norm": 0.01484091062151004,
+      "kl": 0.2568359375,
+      "learning_rate": 8.404e-07,
+      "loss": 0.0003,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1197
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 92.109375,
+      "epoch": 1.9167999999999998,
+      "grad_norm": 5.768276318389724,
+      "kl": 0.2607421875,
+      "learning_rate": 8.402666666666667e-07,
+      "loss": 0.0003,
+      "reward": 1.59375,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.59375,
+      "step": 1198
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 97.84375,
+      "epoch": 1.9184,
+      "grad_norm": 0.05779873562627424,
+      "kl": 0.2412109375,
+      "learning_rate": 8.401333333333332e-07,
+      "loss": 0.0002,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1199
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 95.921875,
+      "epoch": 1.92,
+      "grad_norm": 0.015058047360471977,
+      "kl": 0.23046875,
+      "learning_rate": 8.399999999999999e-07,
+      "loss": 0.0002,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1200
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 103.421875,
+      "epoch": 1.9216,
+      "grad_norm": 1.2032637442775218,
+      "kl": 0.25,
+      "learning_rate": 8.398666666666666e-07,
+      "loss": 0.0003,
+      "reward": 1.71875,
+      "reward_std": 0.0883883461356163,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.734375,
+      "step": 1201
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 94.453125,
+      "epoch": 1.9232,
+      "grad_norm": 27.290166026306157,
+      "kl": 0.2431640625,
+      "learning_rate": 8.397333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.40625,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.40625,
+      "step": 1202
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 104.34375,
+      "epoch": 1.9247999999999998,
+      "grad_norm": 0.013826923771101084,
+      "kl": 0.228515625,
+      "learning_rate": 8.396e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1203
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 96.109375,
+      "epoch": 1.9264000000000001,
+      "grad_norm": 0.011489152875822873,
+      "kl": 0.23193359375,
+      "learning_rate": 8.394666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1204
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 101.234375,
+      "epoch": 1.928,
+      "grad_norm": 0.013596098803107621,
+      "kl": 0.24072265625,
+      "learning_rate": 8.393333333333334e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1205
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 102.9375,
+      "epoch": 1.9296,
+      "grad_norm": 0.012636938009310638,
+      "kl": 0.24853515625,
+      "learning_rate": 8.391999999999999e-07,
+      "loss": 0.0002,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1206
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 96.875,
+      "epoch": 1.9312,
+      "grad_norm": 0.013975767106849536,
+      "kl": 0.24658203125,
+      "learning_rate": 8.390666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1207
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 100.796875,
+      "epoch": 1.9327999999999999,
+      "grad_norm": 0.015368209927213806,
+      "kl": 0.244140625,
+      "learning_rate": 8.389333333333332e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1208
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 100.59375,
+      "epoch": 1.9344000000000001,
+      "grad_norm": 14.36173508984956,
+      "kl": 0.2451171875,
+      "learning_rate": 8.387999999999999e-07,
+      "loss": 0.0002,
+      "reward": 1.71875,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.71875,
+      "step": 1209
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 98.109375,
+      "epoch": 1.936,
+      "grad_norm": 0.013862647212439969,
+      "kl": 0.232421875,
+      "learning_rate": 8.386666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1210
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 99.234375,
+      "epoch": 1.9376,
+      "grad_norm": 13.778325219791345,
+      "kl": 0.24267578125,
+      "learning_rate": 8.385333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.859375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.859375,
+      "step": 1211
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 102.5625,
+      "epoch": 1.9392,
+      "grad_norm": 0.011714023473698654,
+      "kl": 0.23583984375,
+      "learning_rate": 8.384e-07,
+      "loss": 0.0002,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1212
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 103.140625,
+      "epoch": 1.9407999999999999,
+      "grad_norm": 0.011995938471589423,
+      "kl": 0.23876953125,
+      "learning_rate": 8.382666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1213
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 101.46875,
+      "epoch": 1.9424000000000001,
+      "grad_norm": 0.03250316000030743,
+      "kl": 0.24609375,
+      "learning_rate": 8.381333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1214
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 103.875,
+      "epoch": 1.944,
+      "grad_norm": 0.020709323278363643,
+      "kl": 0.23046875,
+      "learning_rate": 8.38e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1215
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 101.984375,
+      "epoch": 1.9456,
+      "grad_norm": 0.013053729320570806,
+      "kl": 0.248046875,
+      "learning_rate": 8.378666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1216
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 98.71875,
+      "epoch": 1.9472,
+      "grad_norm": 0.013894333082104056,
+      "kl": 0.24169921875,
+      "learning_rate": 8.377333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1217
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.3125,
+      "epoch": 1.9487999999999999,
+      "grad_norm": 0.011722837074241342,
+      "kl": 0.21923828125,
+      "learning_rate": 8.375999999999999e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1218
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 103.0625,
+      "epoch": 1.9504000000000001,
+      "grad_norm": 0.0108119738654518,
+      "kl": 0.22509765625,
+      "learning_rate": 8.374666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1219
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 98.890625,
+      "epoch": 1.952,
+      "grad_norm": 0.017578931076109427,
+      "kl": 0.24462890625,
+      "learning_rate": 8.373333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1220
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.453125,
+      "epoch": 1.9536,
+      "grad_norm": 16.648387678336988,
+      "kl": 0.22900390625,
+      "learning_rate": 8.372e-07,
+      "loss": 0.0002,
+      "reward": 1.359375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.359375,
+      "step": 1221
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.3125,
+      "epoch": 1.9552,
+      "grad_norm": 15.814429423490886,
+      "kl": 0.22509765625,
+      "learning_rate": 8.370666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.59375,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.59375,
+      "step": 1222
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 102.703125,
+      "epoch": 1.9567999999999999,
+      "grad_norm": 0.04880716143222236,
+      "kl": 0.2314453125,
+      "learning_rate": 8.369333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1223
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 102.515625,
+      "epoch": 1.9584000000000001,
+      "grad_norm": 0.013790419905554307,
+      "kl": 0.2353515625,
+      "learning_rate": 8.368e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1224
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.09375,
+      "epoch": 1.96,
+      "grad_norm": 0.012165541752923632,
+      "kl": 0.220703125,
+      "learning_rate": 8.366666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1225
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 100.75,
+      "epoch": 1.9616,
+      "grad_norm": 16.03527327816088,
+      "kl": 0.22314453125,
+      "learning_rate": 8.365333333333334e-07,
+      "loss": 0.0002,
+      "reward": 1.96875,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.96875,
+      "step": 1226
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.5625,
+      "epoch": 1.9632,
+      "grad_norm": 13.88960410240311,
+      "kl": 0.22412109375,
+      "learning_rate": 8.363999999999999e-07,
+      "loss": 0.0002,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 1227
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.828125,
+      "epoch": 1.9647999999999999,
+      "grad_norm": 0.01580499187252252,
+      "kl": 0.21337890625,
+      "learning_rate": 8.362666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1228
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 102.984375,
+      "epoch": 1.9664000000000001,
+      "grad_norm": 163.64411582730924,
+      "kl": 0.2392578125,
+      "learning_rate": 8.361333333333332e-07,
+      "loss": 0.0002,
+      "reward": 1.53125,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.53125,
+      "step": 1229
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.578125,
+      "epoch": 1.968,
+      "grad_norm": 0.014193673724873793,
+      "kl": 0.2158203125,
+      "learning_rate": 8.359999999999999e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1230
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.046875,
+      "epoch": 1.9696,
+      "grad_norm": 0.01105913221791902,
+      "kl": 0.2255859375,
+      "learning_rate": 8.358666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1231
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.109375,
+      "epoch": 1.9712,
+      "grad_norm": 0.019282110804204503,
+      "kl": 0.21533203125,
+      "learning_rate": 8.357333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1232
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.421875,
+      "epoch": 1.9727999999999999,
+      "grad_norm": 0.030734218495047833,
+      "kl": 0.234375,
+      "learning_rate": 8.356e-07,
+      "loss": 0.0002,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1233
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.90625,
+      "epoch": 1.9744000000000002,
+      "grad_norm": 0.012075693937411496,
+      "kl": 0.2177734375,
+      "learning_rate": 8.354666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1234
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.515625,
+      "epoch": 1.976,
+      "grad_norm": 0.010528461004064,
+      "kl": 0.21337890625,
+      "learning_rate": 8.353333333333334e-07,
+      "loss": 0.0002,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1235
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 103.40625,
+      "epoch": 1.9776,
+      "grad_norm": 9.141084240140087,
+      "kl": 0.20654296875,
+      "learning_rate": 8.352000000000001e-07,
+      "loss": 0.0002,
+      "reward": 1.859375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.859375,
+      "step": 1236
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.078125,
+      "epoch": 1.9792,
+      "grad_norm": 0.010631269414913366,
+      "kl": 0.216796875,
+      "learning_rate": 8.350666666666665e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1237
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.25,
+      "epoch": 1.9808,
+      "grad_norm": 0.01489042605816367,
+      "kl": 0.2197265625,
+      "learning_rate": 8.349333333333332e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1238
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 103.9375,
+      "epoch": 1.9824000000000002,
+      "grad_norm": 0.013252328197080979,
+      "kl": 0.22314453125,
+      "learning_rate": 8.347999999999999e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1239
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 104.515625,
+      "epoch": 1.984,
+      "grad_norm": 0.012025319927170732,
+      "kl": 0.20166015625,
+      "learning_rate": 8.346666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1240
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.296875,
+      "epoch": 1.9856,
+      "grad_norm": 0.010440250902235404,
+      "kl": 0.20751953125,
+      "learning_rate": 8.345333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1241
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.234375,
+      "epoch": 1.9872,
+      "grad_norm": 0.01150422895888859,
+      "kl": 0.20361328125,
+      "learning_rate": 8.344e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1242
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.0625,
+      "epoch": 1.9888,
+      "grad_norm": 0.015469231286360435,
+      "kl": 0.2109375,
+      "learning_rate": 8.342666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1243
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.84375,
+      "epoch": 1.9904,
+      "grad_norm": 0.009835547755868639,
+      "kl": 0.19677734375,
+      "learning_rate": 8.341333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1244
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.75,
+      "epoch": 1.992,
+      "grad_norm": 25.844678246403475,
+      "kl": 0.197265625,
+      "learning_rate": 8.34e-07,
+      "loss": 0.0002,
+      "reward": 1.71875,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.71875,
+      "step": 1245
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 101.65625,
+      "epoch": 1.9936,
+      "grad_norm": 1.055772905592287,
+      "kl": 0.20458984375,
+      "learning_rate": 8.338666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.75,
+      "step": 1246
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 100.328125,
+      "epoch": 1.9952,
+      "grad_norm": 0.010527820964826667,
+      "kl": 0.18896484375,
+      "learning_rate": 8.337333333333333e-07,
+      "loss": 0.0002,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 1247
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.546875,
+      "epoch": 1.9968,
+      "grad_norm": 0.03230791919184747,
+      "kl": 0.20751953125,
+      "learning_rate": 8.335999999999999e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1248
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.328125,
+      "epoch": 1.9984,
+      "grad_norm": 6.08507115529989,
+      "kl": 0.19677734375,
+      "learning_rate": 8.334666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.828125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.828125,
+      "step": 1249
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.25,
+      "epoch": 2.0,
+      "grad_norm": 0.097858849768676,
+      "kl": 0.18359375,
+      "learning_rate": 8.333333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1250
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.390625,
+      "epoch": 2.0016,
+      "grad_norm": 0.011077305771364643,
+      "kl": 0.205078125,
+      "learning_rate": 8.332e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1251
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.9375,
+      "epoch": 2.0032,
+      "grad_norm": 0.017485244938091626,
+      "kl": 0.205078125,
+      "learning_rate": 8.330666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1252
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.015625,
+      "epoch": 2.0048,
+      "grad_norm": 0.011427960006021273,
+      "kl": 0.1962890625,
+      "learning_rate": 8.329333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1253
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.875,
+      "epoch": 2.0064,
+      "grad_norm": 0.011427833407765342,
+      "kl": 0.19873046875,
+      "learning_rate": 8.328e-07,
+      "loss": 0.0002,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1254
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.109375,
+      "epoch": 2.008,
+      "grad_norm": 0.01455540225986831,
+      "kl": 0.1923828125,
+      "learning_rate": 8.326666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1255
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.890625,
+      "epoch": 2.0096,
+      "grad_norm": 0.013568061681934925,
+      "kl": 0.19775390625,
+      "learning_rate": 8.325333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1256
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.640625,
+      "epoch": 2.0112,
+      "grad_norm": 0.011045428313828976,
+      "kl": 0.18701171875,
+      "learning_rate": 8.324e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1257
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.125,
+      "epoch": 2.0128,
+      "grad_norm": 0.011128817784816789,
+      "kl": 0.20361328125,
+      "learning_rate": 8.322666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1258
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.796875,
+      "epoch": 2.0144,
+      "grad_norm": 0.010138316757284202,
+      "kl": 0.19287109375,
+      "learning_rate": 8.321333333333332e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1259
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.578125,
+      "epoch": 2.016,
+      "grad_norm": 0.01054455163496942,
+      "kl": 0.2041015625,
+      "learning_rate": 8.319999999999999e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1260
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.703125,
+      "epoch": 2.0176,
+      "grad_norm": 0.009688844179358449,
+      "kl": 0.18994140625,
+      "learning_rate": 8.318666666666666e-07,
+      "loss": 0.0002,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 1261
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.125,
+      "epoch": 2.0192,
+      "grad_norm": 52.138073214698416,
+      "kl": 0.2099609375,
+      "learning_rate": 8.317333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.59375,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.59375,
+      "step": 1262
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.96875,
+      "epoch": 2.0208,
+      "grad_norm": 0.012717932632312405,
+      "kl": 0.18994140625,
+      "learning_rate": 8.316e-07,
+      "loss": 0.0002,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1263
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.734375,
+      "epoch": 2.0224,
+      "grad_norm": 0.010624816734439766,
+      "kl": 0.18896484375,
+      "learning_rate": 8.314666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1264
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.640625,
+      "epoch": 2.024,
+      "grad_norm": 0.010485139317047979,
+      "kl": 0.1962890625,
+      "learning_rate": 8.313333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1265
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.28125,
+      "epoch": 2.0256,
+      "grad_norm": 0.015385481654535748,
+      "kl": 0.18359375,
+      "learning_rate": 8.312e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1266
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.78125,
+      "epoch": 2.0272,
+      "grad_norm": 29.291012914455063,
+      "kl": 0.1923828125,
+      "learning_rate": 8.310666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.796875,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.796875,
+      "step": 1267
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.859375,
+      "epoch": 2.0288,
+      "grad_norm": 8.246123876548847,
+      "kl": 0.18603515625,
+      "learning_rate": 8.309333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.671875,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.671875,
+      "step": 1268
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.96875,
+      "epoch": 2.0304,
+      "grad_norm": 0.01093468870093215,
+      "kl": 0.1962890625,
+      "learning_rate": 8.308e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1269
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.171875,
+      "epoch": 2.032,
+      "grad_norm": 0.009966776157496353,
+      "kl": 0.19189453125,
+      "learning_rate": 8.306666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1270
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 103.125,
+      "epoch": 2.0336,
+      "grad_norm": 0.010650390866583554,
+      "kl": 0.19189453125,
+      "learning_rate": 8.305333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1271
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.84375,
+      "epoch": 2.0352,
+      "grad_norm": 0.010051931801185606,
+      "kl": 0.17724609375,
+      "learning_rate": 8.304e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1272
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.15625,
+      "epoch": 2.0368,
+      "grad_norm": 0.012473246428396253,
+      "kl": 0.18505859375,
+      "learning_rate": 8.302666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1273
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.59375,
+      "epoch": 2.0384,
+      "grad_norm": 0.016065877053474835,
+      "kl": 0.19873046875,
+      "learning_rate": 8.301333333333332e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1274
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.71875,
+      "epoch": 2.04,
+      "grad_norm": 0.009556212165589762,
+      "kl": 0.18359375,
+      "learning_rate": 8.299999999999999e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1275
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.0,
+      "epoch": 2.0416,
+      "grad_norm": 0.010237061757324816,
+      "kl": 0.18505859375,
+      "learning_rate": 8.298666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1276
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.15625,
+      "epoch": 2.0432,
+      "grad_norm": 0.011020666462957117,
+      "kl": 0.1865234375,
+      "learning_rate": 8.297333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1277
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.859375,
+      "epoch": 2.0448,
+      "grad_norm": 0.01646466855579025,
+      "kl": 0.19921875,
+      "learning_rate": 8.296e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1278
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.390625,
+      "epoch": 2.0464,
+      "grad_norm": 0.01025065353699701,
+      "kl": 0.18212890625,
+      "learning_rate": 8.294666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1279
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.8125,
+      "epoch": 2.048,
+      "grad_norm": 0.010935327926236043,
+      "kl": 0.1787109375,
+      "learning_rate": 8.293333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1280
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.390625,
+      "epoch": 2.0496,
+      "grad_norm": 0.011039423141693363,
+      "kl": 0.1904296875,
+      "learning_rate": 8.292e-07,
+      "loss": 0.0002,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1281
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.8125,
+      "epoch": 2.0512,
+      "grad_norm": 0.010116866393555026,
+      "kl": 0.181640625,
+      "learning_rate": 8.290666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1282
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.34375,
+      "epoch": 2.0528,
+      "grad_norm": 0.017158855409309066,
+      "kl": 0.2080078125,
+      "learning_rate": 8.289333333333332e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1283
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.75,
+      "epoch": 2.0544,
+      "grad_norm": 0.010528460920032704,
+      "kl": 0.1748046875,
+      "learning_rate": 8.287999999999999e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1284
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.640625,
+      "epoch": 2.056,
+      "grad_norm": 0.01853248333313056,
+      "kl": 0.1669921875,
+      "learning_rate": 8.286666666666666e-07,
+      "loss": 0.0002,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 1285
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.953125,
+      "epoch": 2.0576,
+      "grad_norm": 0.010736555687107905,
+      "kl": 0.17724609375,
+      "learning_rate": 8.285333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1286
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.8125,
+      "epoch": 2.0592,
+      "grad_norm": 21.994410374005092,
+      "kl": 0.1806640625,
+      "learning_rate": 8.284e-07,
+      "loss": 0.0002,
+      "reward": 1.578125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.578125,
+      "step": 1287
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.109375,
+      "epoch": 2.0608,
+      "grad_norm": 0.015207691812966743,
+      "kl": 0.19384765625,
+      "learning_rate": 8.282666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1288
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.734375,
+      "epoch": 2.0624,
+      "grad_norm": 12.166285911979992,
+      "kl": 0.1787109375,
+      "learning_rate": 8.281333333333334e-07,
+      "loss": 0.0002,
+      "reward": 1.84375,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.84375,
+      "step": 1289
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.171875,
+      "epoch": 2.064,
+      "grad_norm": 0.022656923353919986,
+      "kl": 0.1796875,
+      "learning_rate": 8.28e-07,
+      "loss": 0.0002,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1290
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.875,
+      "epoch": 2.0656,
+      "grad_norm": 0.022217420746384168,
+      "kl": 0.1640625,
+      "learning_rate": 8.278666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1291
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.953125,
+      "epoch": 2.0672,
+      "grad_norm": 2.64575098678791,
+      "kl": 0.1728515625,
+      "learning_rate": 8.277333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.78125,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.78125,
+      "step": 1292
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.59375,
+      "epoch": 2.0688,
+      "grad_norm": 0.009478140304301707,
+      "kl": 0.17724609375,
+      "learning_rate": 8.275999999999999e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1293
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.359375,
+      "epoch": 2.0704,
+      "grad_norm": 0.00915324464247392,
+      "kl": 0.171875,
+      "learning_rate": 8.274666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1294
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.640625,
+      "epoch": 2.072,
+      "grad_norm": 0.01011560825823171,
+      "kl": 0.169921875,
+      "learning_rate": 8.273333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1295
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.375,
+      "epoch": 2.0736,
+      "grad_norm": 0.008502385273343904,
+      "kl": 0.17041015625,
+      "learning_rate": 8.272e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1296
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.171875,
+      "epoch": 2.0752,
+      "grad_norm": 0.009414571465774778,
+      "kl": 0.1728515625,
+      "learning_rate": 8.270666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1297
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.5625,
+      "epoch": 2.0768,
+      "grad_norm": 0.014220942840735943,
+      "kl": 0.1689453125,
+      "learning_rate": 8.269333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.25,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.25,
+      "step": 1298
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.0625,
+      "epoch": 2.0784,
+      "grad_norm": 0.011082679334972268,
+      "kl": 0.166015625,
+      "learning_rate": 8.268e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1299
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.109375,
+      "epoch": 2.08,
+      "grad_norm": 0.009316097618276299,
+      "kl": 0.166015625,
+      "learning_rate": 8.266666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1300
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.4375,
+      "epoch": 2.0816,
+      "grad_norm": 0.010522669775267718,
+      "kl": 0.1689453125,
+      "learning_rate": 8.265333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1301
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.4375,
+      "epoch": 2.0832,
+      "grad_norm": 0.008539156148303279,
+      "kl": 0.158203125,
+      "learning_rate": 8.263999999999999e-07,
+      "loss": 0.0002,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1302
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.421875,
+      "epoch": 2.0848,
+      "grad_norm": 0.009060920781578073,
+      "kl": 0.1708984375,
+      "learning_rate": 8.262666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1303
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.703125,
+      "epoch": 2.0864,
+      "grad_norm": 0.009411937214269,
+      "kl": 0.16357421875,
+      "learning_rate": 8.261333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.25,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.25,
+      "step": 1304
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.09375,
+      "epoch": 2.088,
+      "grad_norm": 10.183962383417239,
+      "kl": 0.16357421875,
+      "learning_rate": 8.259999999999999e-07,
+      "loss": 0.0002,
+      "reward": 1.71875,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.71875,
+      "step": 1305
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.65625,
+      "epoch": 2.0896,
+      "grad_norm": 0.014655454439303417,
+      "kl": 0.15478515625,
+      "learning_rate": 8.258666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1306
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.984375,
+      "epoch": 2.0912,
+      "grad_norm": 0.00954438647722795,
+      "kl": 0.1806640625,
+      "learning_rate": 8.257333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1307
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.578125,
+      "epoch": 2.0928,
+      "grad_norm": 0.010980873172480128,
+      "kl": 0.1611328125,
+      "learning_rate": 8.256e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1308
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.6875,
+      "epoch": 2.0944,
+      "grad_norm": 0.011239935240511418,
+      "kl": 0.16162109375,
+      "learning_rate": 8.254666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1309
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.0625,
+      "epoch": 2.096,
+      "grad_norm": 0.010584919908792956,
+      "kl": 0.16552734375,
+      "learning_rate": 8.253333333333334e-07,
+      "loss": 0.0002,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1310
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.234375,
+      "epoch": 2.0976,
+      "grad_norm": 0.009587804026616402,
+      "kl": 0.1611328125,
+      "learning_rate": 8.252000000000001e-07,
+      "loss": 0.0002,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1311
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.203125,
+      "epoch": 2.0992,
+      "grad_norm": 0.011196750910422902,
+      "kl": 0.158203125,
+      "learning_rate": 8.250666666666665e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1312
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.171875,
+      "epoch": 2.1008,
+      "grad_norm": 0.011255231518259957,
+      "kl": 0.15576171875,
+      "learning_rate": 8.249333333333332e-07,
+      "loss": 0.0002,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1313
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.78125,
+      "epoch": 2.1024,
+      "grad_norm": 0.011933744210645057,
+      "kl": 0.15673828125,
+      "learning_rate": 8.247999999999999e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1314
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.421875,
+      "epoch": 2.104,
+      "grad_norm": 0.16125173911753599,
+      "kl": 0.1630859375,
+      "learning_rate": 8.246666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1315
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.015625,
+      "epoch": 2.1056,
+      "grad_norm": 0.009231203109779292,
+      "kl": 0.1513671875,
+      "learning_rate": 8.245333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1316
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.8125,
+      "epoch": 2.1072,
+      "grad_norm": 0.008800556697797156,
+      "kl": 0.1376953125,
+      "learning_rate": 8.244e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1317
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.296875,
+      "epoch": 2.1088,
+      "grad_norm": 1.5932109510338435,
+      "kl": 0.13525390625,
+      "learning_rate": 8.242666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.859375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.859375,
+      "step": 1318
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.734375,
+      "epoch": 2.1104,
+      "grad_norm": 0.009134536010203947,
+      "kl": 0.14697265625,
+      "learning_rate": 8.241333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1319
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.015625,
+      "epoch": 2.112,
+      "grad_norm": 0.010808572586558233,
+      "kl": 0.14501953125,
+      "learning_rate": 8.24e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1320
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.5,
+      "epoch": 2.1136,
+      "grad_norm": 0.016066160888356027,
+      "kl": 0.1474609375,
+      "learning_rate": 8.238666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1321
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.59375,
+      "epoch": 2.1152,
+      "grad_norm": 0.011383154665061653,
+      "kl": 0.1494140625,
+      "learning_rate": 8.237333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.25,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.25,
+      "step": 1322
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.65625,
+      "epoch": 2.1168,
+      "grad_norm": 0.009443327999624393,
+      "kl": 0.14599609375,
+      "learning_rate": 8.235999999999999e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 1323
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.28125,
+      "epoch": 2.1184,
+      "grad_norm": 6.201080725148014,
+      "kl": 0.1484375,
+      "learning_rate": 8.234666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5625,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5625,
+      "step": 1324
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.03125,
+      "epoch": 2.12,
+      "grad_norm": 0.009189670677959274,
+      "kl": 0.14794921875,
+      "learning_rate": 8.233333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1325
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.046875,
+      "epoch": 2.1216,
+      "grad_norm": 46.19025982901258,
+      "kl": 0.15234375,
+      "learning_rate": 8.232e-07,
+      "loss": 0.0002,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 1326
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.890625,
+      "epoch": 2.1232,
+      "grad_norm": 0.009484601809025245,
+      "kl": 0.15625,
+      "learning_rate": 8.230666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1327
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.375,
+      "epoch": 2.1248,
+      "grad_norm": 0.009398070436705699,
+      "kl": 0.166015625,
+      "learning_rate": 8.229333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.25,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.25,
+      "step": 1328
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.46875,
+      "epoch": 2.1264,
+      "grad_norm": 0.009722866102691869,
+      "kl": 0.1689453125,
+      "learning_rate": 8.228e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1329
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.578125,
+      "epoch": 2.128,
+      "grad_norm": 0.009748306178058542,
+      "kl": 0.16015625,
+      "learning_rate": 8.226666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1330
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.109375,
+      "epoch": 2.1296,
+      "grad_norm": 0.009000195355529586,
+      "kl": 0.1533203125,
+      "learning_rate": 8.225333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1331
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.625,
+      "epoch": 2.1312,
+      "grad_norm": 0.00887777254542838,
+      "kl": 0.1630859375,
+      "learning_rate": 8.224e-07,
+      "loss": 0.0002,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1332
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.203125,
+      "epoch": 2.1328,
+      "grad_norm": 0.009568459164340758,
+      "kl": 0.158203125,
+      "learning_rate": 8.222666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1333
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.109375,
+      "epoch": 2.1344,
+      "grad_norm": 0.012913100375020367,
+      "kl": 0.158203125,
+      "learning_rate": 8.221333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1334
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.8125,
+      "epoch": 2.136,
+      "grad_norm": 0.012409374828293778,
+      "kl": 0.16357421875,
+      "learning_rate": 8.219999999999999e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1335
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.171875,
+      "epoch": 2.1376,
+      "grad_norm": 0.008624093855293707,
+      "kl": 0.16064453125,
+      "learning_rate": 8.218666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1336
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.953125,
+      "epoch": 2.1391999999999998,
+      "grad_norm": 0.015656316522331095,
+      "kl": 0.1611328125,
+      "learning_rate": 8.217333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1337
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.71875,
+      "epoch": 2.1408,
+      "grad_norm": 8.961876054812631,
+      "kl": 0.18896484375,
+      "learning_rate": 8.216e-07,
+      "loss": 0.0002,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 1338
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.046875,
+      "epoch": 2.1424,
+      "grad_norm": 0.00961503323664546,
+      "kl": 0.16357421875,
+      "learning_rate": 8.214666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1339
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.109375,
+      "epoch": 2.144,
+      "grad_norm": 1.5418012398068874,
+      "kl": 0.150390625,
+      "learning_rate": 8.213333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.71875,
+      "reward_std": 0.0883883461356163,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.734375,
+      "step": 1340
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.265625,
+      "epoch": 2.1456,
+      "grad_norm": 0.008935605474205162,
+      "kl": 0.1640625,
+      "learning_rate": 8.212e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1341
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.484375,
+      "epoch": 2.1471999999999998,
+      "grad_norm": 0.21068187827381282,
+      "kl": 0.158203125,
+      "learning_rate": 8.210666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.125,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.125,
+      "step": 1342
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.84375,
+      "epoch": 2.1488,
+      "grad_norm": 13.016057237039504,
+      "kl": 0.158203125,
+      "learning_rate": 8.209333333333332e-07,
+      "loss": 0.0002,
+      "reward": 1.546875,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.546875,
+      "step": 1343
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.21875,
+      "epoch": 2.1504,
+      "grad_norm": 0.010832874079836467,
+      "kl": 0.158203125,
+      "learning_rate": 8.207999999999999e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1344
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.265625,
+      "epoch": 2.152,
+      "grad_norm": 1.2008353179284423,
+      "kl": 0.14990234375,
+      "learning_rate": 8.206666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.84375,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.84375,
+      "step": 1345
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.765625,
+      "epoch": 2.1536,
+      "grad_norm": 0.02577293129647609,
+      "kl": 0.14501953125,
+      "learning_rate": 8.205333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1346
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.421875,
+      "epoch": 2.1552,
+      "grad_norm": 0.01180499240349269,
+      "kl": 0.13916015625,
+      "learning_rate": 8.204e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1347
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.671875,
+      "epoch": 2.1568,
+      "grad_norm": 0.017777819764557223,
+      "kl": 0.16796875,
+      "learning_rate": 8.202666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1348
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.703125,
+      "epoch": 2.1584,
+      "grad_norm": 0.009150213344547365,
+      "kl": 0.1455078125,
+      "learning_rate": 8.201333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1349
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 141.84375,
+      "epoch": 2.16,
+      "grad_norm": 0.02001154287928685,
+      "kl": 0.14990234375,
+      "learning_rate": 8.199999999999999e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 1350
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.28125,
+      "epoch": 2.1616,
+      "grad_norm": 10.222691836552038,
+      "kl": 0.13525390625,
+      "learning_rate": 8.198666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.34375,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.34375,
+      "step": 1351
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 150.875,
+      "epoch": 2.1632,
+      "grad_norm": 0.01834565334362387,
+      "kl": 0.13720703125,
+      "learning_rate": 8.197333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1352
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 151.859375,
+      "epoch": 2.1648,
+      "grad_norm": 0.01452372289868314,
+      "kl": 0.13916015625,
+      "learning_rate": 8.196e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1353
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 156.109375,
+      "epoch": 2.1664,
+      "grad_norm": 0.008571491199253016,
+      "kl": 0.13623046875,
+      "learning_rate": 8.194666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1354
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 151.640625,
+      "epoch": 2.168,
+      "grad_norm": 0.00928051236112689,
+      "kl": 0.14697265625,
+      "learning_rate": 8.193333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1355
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 156.09375,
+      "epoch": 2.1696,
+      "grad_norm": 0.008518008881091405,
+      "kl": 0.1337890625,
+      "learning_rate": 8.192e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1356
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 160.265625,
+      "epoch": 2.1712,
+      "grad_norm": 8.377281458878603,
+      "kl": 0.13720703125,
+      "learning_rate": 8.190666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 1357
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 154.359375,
+      "epoch": 2.1728,
+      "grad_norm": 0.04933583978762182,
+      "kl": 0.1572265625,
+      "learning_rate": 8.189333333333332e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1358
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 161.421875,
+      "epoch": 2.1744,
+      "grad_norm": 0.010033578413329077,
+      "kl": 0.1357421875,
+      "learning_rate": 8.187999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1359
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 159.6875,
+      "epoch": 2.176,
+      "grad_norm": 1.1144069197378759,
+      "kl": 0.1416015625,
+      "learning_rate": 8.186666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.609375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.625,
+      "step": 1360
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 166.796875,
+      "epoch": 2.1776,
+      "grad_norm": 10.20623995086869,
+      "kl": 0.14501953125,
+      "learning_rate": 8.185333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.609375,
+      "reward_std": 0.13258251920342445,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.625,
+      "step": 1361
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 159.9375,
+      "epoch": 2.1792,
+      "grad_norm": 0.008296180431982467,
+      "kl": 0.130859375,
+      "learning_rate": 8.184e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1362
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 160.625,
+      "epoch": 2.1808,
+      "grad_norm": 0.007840795224715985,
+      "kl": 0.12744140625,
+      "learning_rate": 8.182666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1363
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 158.734375,
+      "epoch": 2.1824,
+      "grad_norm": 0.008589928110564625,
+      "kl": 0.13818359375,
+      "learning_rate": 8.181333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1364
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 163.296875,
+      "epoch": 2.184,
+      "grad_norm": 0.009149930609377415,
+      "kl": 0.13427734375,
+      "learning_rate": 8.179999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1365
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 154.984375,
+      "epoch": 2.1856,
+      "grad_norm": 0.008623034762259537,
+      "kl": 0.14111328125,
+      "learning_rate": 8.178666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1366
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 156.078125,
+      "epoch": 2.1872,
+      "grad_norm": 0.023364333568453254,
+      "kl": 0.1337890625,
+      "learning_rate": 8.177333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1367
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 157.4375,
+      "epoch": 2.1888,
+      "grad_norm": 25.761725462035376,
+      "kl": 0.140625,
+      "learning_rate": 8.175999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.609375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.609375,
+      "step": 1368
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 164.265625,
+      "epoch": 2.1904,
+      "grad_norm": 0.0089336015258382,
+      "kl": 0.123291015625,
+      "learning_rate": 8.174666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1369
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 160.21875,
+      "epoch": 2.192,
+      "grad_norm": 0.011196572940717521,
+      "kl": 0.13916015625,
+      "learning_rate": 8.173333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1370
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 158.484375,
+      "epoch": 2.1936,
+      "grad_norm": 3.524110884590567,
+      "kl": 0.134765625,
+      "learning_rate": 8.172e-07,
+      "loss": 0.0001,
+      "reward": 1.71875,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.71875,
+      "step": 1371
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 157.65625,
+      "epoch": 2.1952,
+      "grad_norm": 0.010140020857891527,
+      "kl": 0.15087890625,
+      "learning_rate": 8.170666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1372
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 151.765625,
+      "epoch": 2.1968,
+      "grad_norm": 0.01024499083960203,
+      "kl": 0.13720703125,
+      "learning_rate": 8.169333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1373
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 155.734375,
+      "epoch": 2.1984,
+      "grad_norm": 0.007761656851767243,
+      "kl": 0.13623046875,
+      "learning_rate": 8.168e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1374
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 155.8125,
+      "epoch": 2.2,
+      "grad_norm": 0.008410567088415439,
+      "kl": 0.1357421875,
+      "learning_rate": 8.166666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1375
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 152.84375,
+      "epoch": 2.2016,
+      "grad_norm": 0.01393363064167657,
+      "kl": 0.13525390625,
+      "learning_rate": 8.165333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1376
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 159.421875,
+      "epoch": 2.2032,
+      "grad_norm": 0.008959826824096835,
+      "kl": 0.13623046875,
+      "learning_rate": 8.163999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1377
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 161.71875,
+      "epoch": 2.2048,
+      "grad_norm": 0.009402950875155216,
+      "kl": 0.1376953125,
+      "learning_rate": 8.162666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1378
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 159.359375,
+      "epoch": 2.2064,
+      "grad_norm": 0.008533788228723855,
+      "kl": 0.1376953125,
+      "learning_rate": 8.161333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1379
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 160.859375,
+      "epoch": 2.208,
+      "grad_norm": 15.053464624677085,
+      "kl": 0.1376953125,
+      "learning_rate": 8.159999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.609375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.609375,
+      "step": 1380
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 152.265625,
+      "epoch": 2.2096,
+      "grad_norm": 0.008963361339950985,
+      "kl": 0.1376953125,
+      "learning_rate": 8.158666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1381
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 157.28125,
+      "epoch": 2.2112,
+      "grad_norm": 0.007840680719894603,
+      "kl": 0.126953125,
+      "learning_rate": 8.157333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1382
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 164.609375,
+      "epoch": 2.2128,
+      "grad_norm": 0.00852432359098207,
+      "kl": 0.13330078125,
+      "learning_rate": 8.156e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1383
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 159.015625,
+      "epoch": 2.2144,
+      "grad_norm": 0.009110777407827671,
+      "kl": 0.13818359375,
+      "learning_rate": 8.154666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1384
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 154.71875,
+      "epoch": 2.216,
+      "grad_norm": 0.008767830719999349,
+      "kl": 0.14404296875,
+      "learning_rate": 8.153333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1385
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 156.28125,
+      "epoch": 2.2176,
+      "grad_norm": 0.008636892087684768,
+      "kl": 0.1298828125,
+      "learning_rate": 8.152e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1386
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 150.796875,
+      "epoch": 2.2192,
+      "grad_norm": 0.008132755884608928,
+      "kl": 0.13525390625,
+      "learning_rate": 8.150666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1387
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 158.15625,
+      "epoch": 2.2208,
+      "grad_norm": 0.9269013430757993,
+      "kl": 0.138671875,
+      "learning_rate": 8.149333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.96875,
+      "reward_std": 0.0883883461356163,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.984375,
+      "step": 1388
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 155.0,
+      "epoch": 2.2224,
+      "grad_norm": 15.940711850922527,
+      "kl": 0.13134765625,
+      "learning_rate": 8.147999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.90625,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.90625,
+      "step": 1389
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 144.359375,
+      "epoch": 2.224,
+      "grad_norm": 5.819753351524062,
+      "kl": 0.12841796875,
+      "learning_rate": 8.146666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.765625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.765625,
+      "step": 1390
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 155.453125,
+      "epoch": 2.2256,
+      "grad_norm": 0.012235851278188856,
+      "kl": 0.13330078125,
+      "learning_rate": 8.145333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1391
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 149.421875,
+      "epoch": 2.2272,
+      "grad_norm": 0.008101763146684246,
+      "kl": 0.13330078125,
+      "learning_rate": 8.144e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1392
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 148.1875,
+      "epoch": 2.2288,
+      "grad_norm": 0.2062922105633658,
+      "kl": 0.14013671875,
+      "learning_rate": 8.142666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1393
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 146.84375,
+      "epoch": 2.2304,
+      "grad_norm": 0.011582787833968345,
+      "kl": 0.138671875,
+      "learning_rate": 8.141333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1394
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 143.4375,
+      "epoch": 2.232,
+      "grad_norm": 38.057886606728395,
+      "kl": 0.14501953125,
+      "learning_rate": 8.14e-07,
+      "loss": 0.0001,
+      "reward": 1.609375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.609375,
+      "step": 1395
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 147.0625,
+      "epoch": 2.2336,
+      "grad_norm": 0.013348839000649832,
+      "kl": 0.13623046875,
+      "learning_rate": 8.138666666666665e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1396
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 145.3125,
+      "epoch": 2.2352,
+      "grad_norm": 0.007909131636549497,
+      "kl": 0.1318359375,
+      "learning_rate": 8.137333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1397
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 144.828125,
+      "epoch": 2.2368,
+      "grad_norm": 0.04127755554636247,
+      "kl": 0.13134765625,
+      "learning_rate": 8.135999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1398
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 143.09375,
+      "epoch": 2.2384,
+      "grad_norm": 0.009538523111204672,
+      "kl": 0.140625,
+      "learning_rate": 8.134666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1399
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.875,
+      "epoch": 2.24,
+      "grad_norm": 15.05597559604253,
+      "kl": 0.14892578125,
+      "learning_rate": 8.133333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.765625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.765625,
+      "step": 1400
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.046875,
+      "epoch": 2.2416,
+      "grad_norm": 0.03032738221728927,
+      "kl": 0.1533203125,
+      "learning_rate": 8.132e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1401
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.09375,
+      "epoch": 2.2432,
+      "grad_norm": 0.008647831089433447,
+      "kl": 0.138671875,
+      "learning_rate": 8.130666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1402
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 141.109375,
+      "epoch": 2.2448,
+      "grad_norm": 0.008557252621588232,
+      "kl": 0.14208984375,
+      "learning_rate": 8.129333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1403
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.546875,
+      "epoch": 2.2464,
+      "grad_norm": 0.012194153688901454,
+      "kl": 0.146484375,
+      "learning_rate": 8.128e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 1404
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.625,
+      "epoch": 2.248,
+      "grad_norm": 0.008764460252640632,
+      "kl": 0.138671875,
+      "learning_rate": 8.126666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1405
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.796875,
+      "epoch": 2.2496,
+      "grad_norm": 0.009328483041786885,
+      "kl": 0.13427734375,
+      "learning_rate": 8.125333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1406
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.375,
+      "epoch": 2.2512,
+      "grad_norm": 9.303809457532571,
+      "kl": 0.1396484375,
+      "learning_rate": 8.123999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.5625,
+      "reward_std": 0.1157275140285492,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5625,
+      "step": 1407
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.984375,
+      "epoch": 2.2528,
+      "grad_norm": 0.012017962483388303,
+      "kl": 0.14404296875,
+      "learning_rate": 8.122666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1408
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 141.265625,
+      "epoch": 2.2544,
+      "grad_norm": 29.79864038958323,
+      "kl": 0.1455078125,
+      "learning_rate": 8.121333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.515625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.515625,
+      "step": 1409
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.296875,
+      "epoch": 2.2560000000000002,
+      "grad_norm": 0.00911650675199813,
+      "kl": 0.13330078125,
+      "learning_rate": 8.12e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1410
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 141.59375,
+      "epoch": 2.2576,
+      "grad_norm": 39.46461678857416,
+      "kl": 0.14306640625,
+      "learning_rate": 8.118666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.65625,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.65625,
+      "step": 1411
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.296875,
+      "epoch": 2.2592,
+      "grad_norm": 0.009622645334549889,
+      "kl": 0.146484375,
+      "learning_rate": 8.117333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1412
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 141.53125,
+      "epoch": 2.2608,
+      "grad_norm": 0.008177037454647347,
+      "kl": 0.12890625,
+      "learning_rate": 8.116e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 1413
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.140625,
+      "epoch": 2.2624,
+      "grad_norm": 3.086085786049874,
+      "kl": 0.13330078125,
+      "learning_rate": 8.114666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.265625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.265625,
+      "step": 1414
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.0625,
+      "epoch": 2.2640000000000002,
+      "grad_norm": 0.030202161917797014,
+      "kl": 0.131591796875,
+      "learning_rate": 8.113333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1415
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 146.046875,
+      "epoch": 2.2656,
+      "grad_norm": 0.012271076120246908,
+      "kl": 0.134765625,
+      "learning_rate": 8.112e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1416
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 141.609375,
+      "epoch": 2.2672,
+      "grad_norm": 0.026761423921491754,
+      "kl": 0.14013671875,
+      "learning_rate": 8.110666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1417
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 146.40625,
+      "epoch": 2.2688,
+      "grad_norm": 20.942219723987094,
+      "kl": 0.13427734375,
+      "learning_rate": 8.109333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.84375,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.84375,
+      "step": 1418
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 149.03125,
+      "epoch": 2.2704,
+      "grad_norm": 12.350605994747394,
+      "kl": 0.14208984375,
+      "learning_rate": 8.107999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.609375,
+      "reward_std": 0.10205793380737305,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.609375,
+      "step": 1419
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 146.296875,
+      "epoch": 2.2720000000000002,
+      "grad_norm": 0.00999236670896326,
+      "kl": 0.129150390625,
+      "learning_rate": 8.106666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1420
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 144.265625,
+      "epoch": 2.2736,
+      "grad_norm": 0.00831607559993021,
+      "kl": 0.14013671875,
+      "learning_rate": 8.105333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1421
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 148.078125,
+      "epoch": 2.2752,
+      "grad_norm": 0.008406546162805565,
+      "kl": 0.115478515625,
+      "learning_rate": 8.104e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1422
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 153.59375,
+      "epoch": 2.2768,
+      "grad_norm": 0.007927152871004673,
+      "kl": 0.124755859375,
+      "learning_rate": 8.102666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1423
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 149.296875,
+      "epoch": 2.2784,
+      "grad_norm": 0.011268575595410347,
+      "kl": 0.14111328125,
+      "learning_rate": 8.101333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1424
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 157.546875,
+      "epoch": 2.2800000000000002,
+      "grad_norm": 0.021620315213147348,
+      "kl": 0.1337890625,
+      "learning_rate": 8.1e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 1425
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 148.875,
+      "epoch": 2.2816,
+      "grad_norm": 0.011138785206757864,
+      "kl": 0.124755859375,
+      "learning_rate": 8.098666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1426
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 149.359375,
+      "epoch": 2.2832,
+      "grad_norm": 5.324954227268451,
+      "kl": 0.12744140625,
+      "learning_rate": 8.097333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.65625,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.65625,
+      "step": 1427
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 155.0,
+      "epoch": 2.2848,
+      "grad_norm": 0.008508145025247733,
+      "kl": 0.119873046875,
+      "learning_rate": 8.095999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1428
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 152.734375,
+      "epoch": 2.2864,
+      "grad_norm": 0.007477541323893755,
+      "kl": 0.12890625,
+      "learning_rate": 8.094666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1429
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 150.875,
+      "epoch": 2.288,
+      "grad_norm": 0.01021461446796877,
+      "kl": 0.12353515625,
+      "learning_rate": 8.093333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1430
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 159.765625,
+      "epoch": 2.2896,
+      "grad_norm": 0.007926830246336355,
+      "kl": 0.123291015625,
+      "learning_rate": 8.092e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1431
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 157.3125,
+      "epoch": 2.2912,
+      "grad_norm": 0.00901553761404747,
+      "kl": 0.12158203125,
+      "learning_rate": 8.090666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1432
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 155.109375,
+      "epoch": 2.2928,
+      "grad_norm": 0.009965641392086444,
+      "kl": 0.11962890625,
+      "learning_rate": 8.089333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1433
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 154.859375,
+      "epoch": 2.2944,
+      "grad_norm": 0.007575566879746791,
+      "kl": 0.126708984375,
+      "learning_rate": 8.087999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1434
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 158.234375,
+      "epoch": 2.296,
+      "grad_norm": 0.02956541534156907,
+      "kl": 0.126220703125,
+      "learning_rate": 8.086666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1435
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 158.484375,
+      "epoch": 2.2976,
+      "grad_norm": 0.007648827689889943,
+      "kl": 0.126953125,
+      "learning_rate": 8.085333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1436
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 155.453125,
+      "epoch": 2.2992,
+      "grad_norm": 0.007415249769491875,
+      "kl": 0.121337890625,
+      "learning_rate": 8.084e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1437
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 153.921875,
+      "epoch": 2.3008,
+      "grad_norm": 0.008742463184350492,
+      "kl": 0.12646484375,
+      "learning_rate": 8.082666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1438
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 150.125,
+      "epoch": 2.3024,
+      "grad_norm": 0.011232349054739866,
+      "kl": 0.117431640625,
+      "learning_rate": 8.081333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1439
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 158.0625,
+      "epoch": 2.304,
+      "grad_norm": 0.013634852928431647,
+      "kl": 0.11767578125,
+      "learning_rate": 8.08e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1440
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 159.53125,
+      "epoch": 2.3056,
+      "grad_norm": 14.329601988607406,
+      "kl": 0.123046875,
+      "learning_rate": 8.078666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 1441
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 166.5,
+      "epoch": 2.3072,
+      "grad_norm": 0.0064936945188490336,
+      "kl": 0.113037109375,
+      "learning_rate": 8.077333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1442
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 161.53125,
+      "epoch": 2.3088,
+      "grad_norm": 0.013406981814436001,
+      "kl": 0.1259765625,
+      "learning_rate": 8.075999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1443
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 162.03125,
+      "epoch": 2.3104,
+      "grad_norm": 0.007579699395969868,
+      "kl": 0.115234375,
+      "learning_rate": 8.074666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1444
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 159.828125,
+      "epoch": 2.312,
+      "grad_norm": 0.014272669335847527,
+      "kl": 0.128662109375,
+      "learning_rate": 8.073333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1445
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 168.890625,
+      "epoch": 2.3136,
+      "grad_norm": 0.006909984474634915,
+      "kl": 0.1181640625,
+      "learning_rate": 8.072e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1446
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 157.296875,
+      "epoch": 2.3152,
+      "grad_norm": 0.008733854542193786,
+      "kl": 0.1142578125,
+      "learning_rate": 8.070666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1447
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 160.734375,
+      "epoch": 2.3168,
+      "grad_norm": 0.015707102173975737,
+      "kl": 0.116943359375,
+      "learning_rate": 8.069333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1448
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 157.625,
+      "epoch": 2.3184,
+      "grad_norm": 0.008664971568497559,
+      "kl": 0.12353515625,
+      "learning_rate": 8.067999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1449
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 160.53125,
+      "epoch": 2.32,
+      "grad_norm": 22.191071115743423,
+      "kl": 0.121337890625,
+      "learning_rate": 8.066666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.703125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.703125,
+      "step": 1450
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 162.625,
+      "epoch": 2.3216,
+      "grad_norm": 0.006822998360411303,
+      "kl": 0.113037109375,
+      "learning_rate": 8.065333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1451
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 159.921875,
+      "epoch": 2.3232,
+      "grad_norm": 0.007585314178914968,
+      "kl": 0.115478515625,
+      "learning_rate": 8.064e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1452
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 163.34375,
+      "epoch": 2.3247999999999998,
+      "grad_norm": 0.007484176488638552,
+      "kl": 0.1220703125,
+      "learning_rate": 8.062666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1453
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 159.5,
+      "epoch": 2.3264,
+      "grad_norm": 4.488974366653136,
+      "kl": 0.10986328125,
+      "learning_rate": 8.061333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.765625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.765625,
+      "step": 1454
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 171.09375,
+      "epoch": 2.328,
+      "grad_norm": 0.007616816873661686,
+      "kl": 0.117431640625,
+      "learning_rate": 8.06e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1455
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 161.484375,
+      "epoch": 2.3296,
+      "grad_norm": 8.387853978882667,
+      "kl": 0.1201171875,
+      "learning_rate": 8.058666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.65625,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.65625,
+      "step": 1456
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 159.65625,
+      "epoch": 2.3312,
+      "grad_norm": 0.00983136217507699,
+      "kl": 0.110595703125,
+      "learning_rate": 8.057333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1457
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 163.21875,
+      "epoch": 2.3327999999999998,
+      "grad_norm": 0.008664875544018543,
+      "kl": 0.113525390625,
+      "learning_rate": 8.056e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1458
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 165.203125,
+      "epoch": 2.3344,
+      "grad_norm": 0.009130758522368772,
+      "kl": 0.112548828125,
+      "learning_rate": 8.054666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1459
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 158.953125,
+      "epoch": 2.336,
+      "grad_norm": 0.007258110212946198,
+      "kl": 0.118408203125,
+      "learning_rate": 8.053333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1460
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 154.9375,
+      "epoch": 2.3376,
+      "grad_norm": 0.006945083332657677,
+      "kl": 0.11279296875,
+      "learning_rate": 8.052e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1461
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 159.03125,
+      "epoch": 2.3392,
+      "grad_norm": 0.00885396527469786,
+      "kl": 0.115966796875,
+      "learning_rate": 8.050666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1462
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 158.6875,
+      "epoch": 2.3407999999999998,
+      "grad_norm": 0.008179703833125643,
+      "kl": 0.116455078125,
+      "learning_rate": 8.049333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1463
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 160.359375,
+      "epoch": 2.3424,
+      "grad_norm": 0.01035712812794954,
+      "kl": 0.1201171875,
+      "learning_rate": 8.047999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1464
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 170.328125,
+      "epoch": 2.344,
+      "grad_norm": 0.9810809805312302,
+      "kl": 0.1201171875,
+      "learning_rate": 8.046666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.59375,
+      "reward_std": 0.0883883461356163,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.609375,
+      "step": 1465
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 160.796875,
+      "epoch": 2.3456,
+      "grad_norm": 0.013657744202215109,
+      "kl": 0.111083984375,
+      "learning_rate": 8.045333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1466
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 155.1875,
+      "epoch": 2.3472,
+      "grad_norm": 0.006917046682469892,
+      "kl": 0.115478515625,
+      "learning_rate": 8.044e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1467
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 153.09375,
+      "epoch": 2.3487999999999998,
+      "grad_norm": 0.008373031550046005,
+      "kl": 0.107421875,
+      "learning_rate": 8.042666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1468
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 149.171875,
+      "epoch": 2.3504,
+      "grad_norm": 60.390625887780004,
+      "kl": 0.113037109375,
+      "learning_rate": 8.041333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.421875,
+      "reward_std": 0.12255740165710449,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.421875,
+      "step": 1469
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 148.5625,
+      "epoch": 2.352,
+      "grad_norm": 0.007597519315208329,
+      "kl": 0.108642578125,
+      "learning_rate": 8.04e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1470
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 150.25,
+      "epoch": 2.3536,
+      "grad_norm": 36.241302264297204,
+      "kl": 0.111572265625,
+      "learning_rate": 8.038666666666665e-07,
+      "loss": 0.0001,
+      "reward": 1.34375,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.34375,
+      "step": 1471
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 147.8125,
+      "epoch": 2.3552,
+      "grad_norm": 48.845050682126505,
+      "kl": 0.1162109375,
+      "learning_rate": 8.037333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.890625,
+      "reward_std": 0.1315089464187622,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.890625,
+      "step": 1472
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 151.625,
+      "epoch": 2.3568,
+      "grad_norm": 0.00863882352788133,
+      "kl": 0.107421875,
+      "learning_rate": 8.035999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1473
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 143.5625,
+      "epoch": 2.3584,
+      "grad_norm": 1.0493055258156043,
+      "kl": 0.10498046875,
+      "learning_rate": 8.034666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.765625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.765625,
+      "step": 1474
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 145.03125,
+      "epoch": 2.36,
+      "grad_norm": 4.370669626060684,
+      "kl": 0.113525390625,
+      "learning_rate": 8.033333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.609375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.609375,
+      "step": 1475
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 142.28125,
+      "epoch": 2.3616,
+      "grad_norm": 0.0152185664195749,
+      "kl": 0.114501953125,
+      "learning_rate": 8.032e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1476
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 146.515625,
+      "epoch": 2.3632,
+      "grad_norm": 1.3492433927685445,
+      "kl": 0.115234375,
+      "learning_rate": 8.030666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.75,
+      "step": 1477
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 142.0625,
+      "epoch": 2.3648,
+      "grad_norm": 0.007353718576992059,
+      "kl": 0.12646484375,
+      "learning_rate": 8.029333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1478
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.796875,
+      "epoch": 2.3664,
+      "grad_norm": 0.008533763120359828,
+      "kl": 0.11669921875,
+      "learning_rate": 8.028e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1479
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 145.875,
+      "epoch": 2.368,
+      "grad_norm": 0.00974978937518155,
+      "kl": 0.140625,
+      "learning_rate": 8.026666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1480
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 142.28125,
+      "epoch": 2.3696,
+      "grad_norm": 0.01095530253704724,
+      "kl": 0.1298828125,
+      "learning_rate": 8.025333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1481
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 148.125,
+      "epoch": 2.3712,
+      "grad_norm": 0.008066226973445126,
+      "kl": 0.120361328125,
+      "learning_rate": 8.023999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1482
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 141.59375,
+      "epoch": 2.3728,
+      "grad_norm": 0.013182812827397312,
+      "kl": 0.12890625,
+      "learning_rate": 8.022666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1483
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 141.46875,
+      "epoch": 2.3744,
+      "grad_norm": 0.007509216436452009,
+      "kl": 0.124267578125,
+      "learning_rate": 8.021333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1484
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 146.015625,
+      "epoch": 2.376,
+      "grad_norm": 0.006980121771039459,
+      "kl": 0.130859375,
+      "learning_rate": 8.02e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1485
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 144.03125,
+      "epoch": 2.3776,
+      "grad_norm": 0.8582553239609826,
+      "kl": 0.127685546875,
+      "learning_rate": 8.018666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.609375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.625,
+      "step": 1486
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.671875,
+      "epoch": 2.3792,
+      "grad_norm": 0.017067404383451962,
+      "kl": 0.1337890625,
+      "learning_rate": 8.017333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.25,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.25,
+      "step": 1487
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.09375,
+      "epoch": 2.3808,
+      "grad_norm": 0.01138617348749591,
+      "kl": 0.1416015625,
+      "learning_rate": 8.016e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1488
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.21875,
+      "epoch": 2.3824,
+      "grad_norm": 0.008962723931320077,
+      "kl": 0.14599609375,
+      "learning_rate": 8.014666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1489
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.15625,
+      "epoch": 2.384,
+      "grad_norm": 0.009053814361128445,
+      "kl": 0.14794921875,
+      "learning_rate": 8.013333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1490
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.84375,
+      "epoch": 2.3856,
+      "grad_norm": 10.010322412909773,
+      "kl": 0.1484375,
+      "learning_rate": 8.012e-07,
+      "loss": 0.0001,
+      "reward": 1.515625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.515625,
+      "step": 1491
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.125,
+      "epoch": 2.3872,
+      "grad_norm": 0.017680785319015548,
+      "kl": 0.1484375,
+      "learning_rate": 8.010666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1492
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.34375,
+      "epoch": 2.3888,
+      "grad_norm": 9.853536634827165,
+      "kl": 0.1552734375,
+      "learning_rate": 8.009333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.65625,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.65625,
+      "step": 1493
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.40625,
+      "epoch": 2.3904,
+      "grad_norm": 0.0084610546090661,
+      "kl": 0.15673828125,
+      "learning_rate": 8.007999999999999e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1494
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.203125,
+      "epoch": 2.392,
+      "grad_norm": 0.009144933949899783,
+      "kl": 0.16455078125,
+      "learning_rate": 8.006666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1495
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.4375,
+      "epoch": 2.3936,
+      "grad_norm": 0.03170785130239686,
+      "kl": 0.1865234375,
+      "learning_rate": 8.005333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1496
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.875,
+      "epoch": 2.3952,
+      "grad_norm": 0.01051776617556539,
+      "kl": 0.1513671875,
+      "learning_rate": 8.004e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1497
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.34375,
+      "epoch": 2.3968,
+      "grad_norm": 1.6870416019829504,
+      "kl": 0.16552734375,
+      "learning_rate": 8.002666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.640625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.640625,
+      "step": 1498
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.890625,
+      "epoch": 2.3984,
+      "grad_norm": 0.012025658651804707,
+      "kl": 0.162109375,
+      "learning_rate": 8.001333333333334e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1499
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.515625,
+      "epoch": 2.4,
+      "grad_norm": 0.014702121944655774,
+      "kl": 0.15966796875,
+      "learning_rate": 8e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1500
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.875,
+      "epoch": 2.4016,
+      "grad_norm": 0.009294810132926788,
+      "kl": 0.16845703125,
+      "learning_rate": 7.998666666666665e-07,
+      "loss": 0.0002,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1501
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.1875,
+      "epoch": 2.4032,
+      "grad_norm": 20.615496202255827,
+      "kl": 0.25390625,
+      "learning_rate": 7.997333333333332e-07,
+      "loss": 0.0003,
+      "reward": 1.859375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.859375,
+      "step": 1502
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.0625,
+      "epoch": 2.4048,
+      "grad_norm": 0.011871779420511552,
+      "kl": 0.16845703125,
+      "learning_rate": 7.995999999999999e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1503
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.171875,
+      "epoch": 2.4064,
+      "grad_norm": 0.023527821478072367,
+      "kl": 0.17724609375,
+      "learning_rate": 7.994666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1504
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.421875,
+      "epoch": 2.408,
+      "grad_norm": 27.65841929004552,
+      "kl": 0.19775390625,
+      "learning_rate": 7.993333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0883883461356163,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1505
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.609375,
+      "epoch": 2.4096,
+      "grad_norm": 8.291378247339225,
+      "kl": 0.15673828125,
+      "learning_rate": 7.992e-07,
+      "loss": 0.0002,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 1506
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.984375,
+      "epoch": 2.4112,
+      "grad_norm": 5.858049929698991,
+      "kl": 0.1572265625,
+      "learning_rate": 7.990666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.859375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.859375,
+      "step": 1507
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.953125,
+      "epoch": 2.4128,
+      "grad_norm": 0.008848638038496857,
+      "kl": 0.1650390625,
+      "learning_rate": 7.989333333333334e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1508
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.234375,
+      "epoch": 2.4144,
+      "grad_norm": 41.74360046125714,
+      "kl": 0.16845703125,
+      "learning_rate": 7.987999999999999e-07,
+      "loss": 0.0002,
+      "reward": 1.9375,
+      "reward_std": 0.10888781771063805,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.9375,
+      "step": 1509
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.4375,
+      "epoch": 2.416,
+      "grad_norm": 15.019391255903164,
+      "kl": 0.169921875,
+      "learning_rate": 7.986666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.640625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.640625,
+      "step": 1510
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.140625,
+      "epoch": 2.4176,
+      "grad_norm": 0.009212005489217004,
+      "kl": 0.1611328125,
+      "learning_rate": 7.985333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1511
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.546875,
+      "epoch": 2.4192,
+      "grad_norm": 0.010597873395413801,
+      "kl": 0.16845703125,
+      "learning_rate": 7.984e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1512
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.90625,
+      "epoch": 2.4208,
+      "grad_norm": 0.009594229993154187,
+      "kl": 0.16064453125,
+      "learning_rate": 7.982666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1513
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.875,
+      "epoch": 2.4224,
+      "grad_norm": 41.354831977209216,
+      "kl": 0.1689453125,
+      "learning_rate": 7.981333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.859375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.859375,
+      "step": 1514
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.84375,
+      "epoch": 2.424,
+      "grad_norm": 0.009877751054066148,
+      "kl": 0.1689453125,
+      "learning_rate": 7.98e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1515
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.90625,
+      "epoch": 2.4256,
+      "grad_norm": 0.012043421434857846,
+      "kl": 0.1650390625,
+      "learning_rate": 7.978666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1516
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.296875,
+      "epoch": 2.4272,
+      "grad_norm": 0.010749133546638286,
+      "kl": 0.1611328125,
+      "learning_rate": 7.977333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1517
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.078125,
+      "epoch": 2.4288,
+      "grad_norm": 0.01393009361367489,
+      "kl": 0.1669921875,
+      "learning_rate": 7.975999999999999e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1518
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.140625,
+      "epoch": 2.4304,
+      "grad_norm": 0.010091539359965214,
+      "kl": 0.16796875,
+      "learning_rate": 7.974666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1519
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.953125,
+      "epoch": 2.432,
+      "grad_norm": 0.013619023965666836,
+      "kl": 0.1728515625,
+      "learning_rate": 7.973333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1520
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.15625,
+      "epoch": 2.4336,
+      "grad_norm": 0.015879439116265777,
+      "kl": 0.17138671875,
+      "learning_rate": 7.972e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1521
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.15625,
+      "epoch": 2.4352,
+      "grad_norm": 8.377334284915374,
+      "kl": 0.1787109375,
+      "learning_rate": 7.970666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.71875,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.71875,
+      "step": 1522
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.921875,
+      "epoch": 2.4368,
+      "grad_norm": 5.538943315661335,
+      "kl": 0.169921875,
+      "learning_rate": 7.969333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.515625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.515625,
+      "step": 1523
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.0,
+      "epoch": 2.4384,
+      "grad_norm": 0.014471224190591547,
+      "kl": 0.1611328125,
+      "learning_rate": 7.967999999999999e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1524
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.640625,
+      "epoch": 2.44,
+      "grad_norm": 0.0162770922946918,
+      "kl": 0.1728515625,
+      "learning_rate": 7.966666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1525
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.859375,
+      "epoch": 2.4416,
+      "grad_norm": 0.012462966632811294,
+      "kl": 0.1591796875,
+      "learning_rate": 7.965333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1526
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.265625,
+      "epoch": 2.4432,
+      "grad_norm": 0.009230641263571444,
+      "kl": 0.15576171875,
+      "learning_rate": 7.964e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1527
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.140625,
+      "epoch": 2.4448,
+      "grad_norm": 0.0724068166095941,
+      "kl": 0.1572265625,
+      "learning_rate": 7.962666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1528
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.453125,
+      "epoch": 2.4464,
+      "grad_norm": 0.011163984311308444,
+      "kl": 0.1533203125,
+      "learning_rate": 7.961333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1529
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.515625,
+      "epoch": 2.448,
+      "grad_norm": 0.012824891119288914,
+      "kl": 0.1552734375,
+      "learning_rate": 7.96e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1530
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.15625,
+      "epoch": 2.4496,
+      "grad_norm": 22.0644634827285,
+      "kl": 0.1640625,
+      "learning_rate": 7.958666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.5625,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5625,
+      "step": 1531
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.46875,
+      "epoch": 2.4512,
+      "grad_norm": 0.008563937927830733,
+      "kl": 0.1435546875,
+      "learning_rate": 7.957333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1532
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.84375,
+      "epoch": 2.4528,
+      "grad_norm": 0.015112717813469305,
+      "kl": 0.154296875,
+      "learning_rate": 7.956e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1533
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.296875,
+      "epoch": 2.4544,
+      "grad_norm": 0.00819376308025162,
+      "kl": 0.154296875,
+      "learning_rate": 7.954666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1534
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.6875,
+      "epoch": 2.456,
+      "grad_norm": 0.020733335734341787,
+      "kl": 0.16552734375,
+      "learning_rate": 7.953333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1535
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.578125,
+      "epoch": 2.4576000000000002,
+      "grad_norm": 0.009426617094508198,
+      "kl": 0.1611328125,
+      "learning_rate": 7.952e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1536
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.03125,
+      "epoch": 2.4592,
+      "grad_norm": 0.012512104475003434,
+      "kl": 0.16064453125,
+      "learning_rate": 7.950666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1537
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.015625,
+      "epoch": 2.4608,
+      "grad_norm": 0.011293798193041955,
+      "kl": 0.1591796875,
+      "learning_rate": 7.949333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1538
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.25,
+      "epoch": 2.4624,
+      "grad_norm": 38.32749542311773,
+      "kl": 0.15576171875,
+      "learning_rate": 7.947999999999999e-07,
+      "loss": 0.0002,
+      "reward": 1.515625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.515625,
+      "step": 1539
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.34375,
+      "epoch": 2.464,
+      "grad_norm": 5.296130720892049,
+      "kl": 0.15576171875,
+      "learning_rate": 7.946666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.640625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.640625,
+      "step": 1540
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.671875,
+      "epoch": 2.4656000000000002,
+      "grad_norm": 0.00905659185253877,
+      "kl": 0.14892578125,
+      "learning_rate": 7.945333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1541
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.890625,
+      "epoch": 2.4672,
+      "grad_norm": 0.009030707524821331,
+      "kl": 0.1455078125,
+      "learning_rate": 7.944e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1542
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.578125,
+      "epoch": 2.4688,
+      "grad_norm": 0.009392558748879301,
+      "kl": 0.14501953125,
+      "learning_rate": 7.942666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1543
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.03125,
+      "epoch": 2.4704,
+      "grad_norm": 0.014335601926066322,
+      "kl": 0.14892578125,
+      "learning_rate": 7.941333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1544
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.8125,
+      "epoch": 2.472,
+      "grad_norm": 0.02161151204535267,
+      "kl": 0.13525390625,
+      "learning_rate": 7.94e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1545
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.6875,
+      "epoch": 2.4736000000000002,
+      "grad_norm": 0.010130664826484196,
+      "kl": 0.15283203125,
+      "learning_rate": 7.938666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1546
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.421875,
+      "epoch": 2.4752,
+      "grad_norm": 8.529296575649225,
+      "kl": 0.14892578125,
+      "learning_rate": 7.937333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.6875,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.6875,
+      "step": 1547
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.984375,
+      "epoch": 2.4768,
+      "grad_norm": 0.009133363853352587,
+      "kl": 0.14208984375,
+      "learning_rate": 7.935999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1548
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.71875,
+      "epoch": 2.4784,
+      "grad_norm": 0.009924365645116399,
+      "kl": 0.1396484375,
+      "learning_rate": 7.934666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1549
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.28125,
+      "epoch": 2.48,
+      "grad_norm": 0.022928717715425425,
+      "kl": 0.14111328125,
+      "learning_rate": 7.933333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1550
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.75,
+      "epoch": 2.4816,
+      "grad_norm": 0.008403917439089357,
+      "kl": 0.1474609375,
+      "learning_rate": 7.932e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1551
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.625,
+      "epoch": 2.4832,
+      "grad_norm": 6.640569351562569,
+      "kl": 0.15283203125,
+      "learning_rate": 7.930666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.703125,
+      "reward_std": 0.10205793008208275,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.703125,
+      "step": 1552
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.203125,
+      "epoch": 2.4848,
+      "grad_norm": 28.48822780965907,
+      "kl": 0.1435546875,
+      "learning_rate": 7.929333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.546875,
+      "reward_std": 0.10205793380737305,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.546875,
+      "step": 1553
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.328125,
+      "epoch": 2.4864,
+      "grad_norm": 0.007710444037873155,
+      "kl": 0.1318359375,
+      "learning_rate": 7.928e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1554
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.4375,
+      "epoch": 2.488,
+      "grad_norm": 0.009988841512614454,
+      "kl": 0.134765625,
+      "learning_rate": 7.926666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1555
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.25,
+      "epoch": 2.4896,
+      "grad_norm": 0.010944524679352452,
+      "kl": 0.1279296875,
+      "learning_rate": 7.925333333333332e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 1556
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.671875,
+      "epoch": 2.4912,
+      "grad_norm": 0.010336496701497586,
+      "kl": 0.13623046875,
+      "learning_rate": 7.923999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1557
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.859375,
+      "epoch": 2.4928,
+      "grad_norm": 0.008564129868859748,
+      "kl": 0.1513671875,
+      "learning_rate": 7.922666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1558
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.4375,
+      "epoch": 2.4944,
+      "grad_norm": 0.007570562876994652,
+      "kl": 0.13037109375,
+      "learning_rate": 7.921333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1559
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.6875,
+      "epoch": 2.496,
+      "grad_norm": 0.007835450672608366,
+      "kl": 0.1259765625,
+      "learning_rate": 7.92e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1560
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.65625,
+      "epoch": 2.4976,
+      "grad_norm": 0.009600492409851162,
+      "kl": 0.13427734375,
+      "learning_rate": 7.918666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1561
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.09375,
+      "epoch": 2.4992,
+      "grad_norm": 0.01656644358827863,
+      "kl": 0.1435546875,
+      "learning_rate": 7.917333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1562
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.140625,
+      "epoch": 2.5008,
+      "grad_norm": 0.01193587284274065,
+      "kl": 0.13134765625,
+      "learning_rate": 7.916e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1563
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.4375,
+      "epoch": 2.5023999999999997,
+      "grad_norm": 0.01096756297435194,
+      "kl": 0.1259765625,
+      "learning_rate": 7.914666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1564
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.921875,
+      "epoch": 2.504,
+      "grad_norm": 41.98220801308427,
+      "kl": 0.14013671875,
+      "learning_rate": 7.913333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.46875,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.46875,
+      "step": 1565
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.890625,
+      "epoch": 2.5056000000000003,
+      "grad_norm": 3.968588548394872,
+      "kl": 0.13720703125,
+      "learning_rate": 7.911999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.609375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.625,
+      "step": 1566
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.234375,
+      "epoch": 2.5072,
+      "grad_norm": 0.008215017778606537,
+      "kl": 0.12841796875,
+      "learning_rate": 7.910666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1567
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.0625,
+      "epoch": 2.5088,
+      "grad_norm": 88.08016905927754,
+      "kl": 0.12939453125,
+      "learning_rate": 7.909333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5625,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5625,
+      "step": 1568
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.984375,
+      "epoch": 2.5103999999999997,
+      "grad_norm": 2.6501037319352236,
+      "kl": 0.12255859375,
+      "learning_rate": 7.907999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.609375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.609375,
+      "step": 1569
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.234375,
+      "epoch": 2.512,
+      "grad_norm": 0.007449467990402051,
+      "kl": 0.13525390625,
+      "learning_rate": 7.906666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1570
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.84375,
+      "epoch": 2.5136,
+      "grad_norm": 0.008513445367631246,
+      "kl": 0.129150390625,
+      "learning_rate": 7.905333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1571
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.4375,
+      "epoch": 2.5152,
+      "grad_norm": 0.007503137114160498,
+      "kl": 0.12548828125,
+      "learning_rate": 7.904e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1572
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.390625,
+      "epoch": 2.5168,
+      "grad_norm": 0.007540464026436219,
+      "kl": 0.131591796875,
+      "learning_rate": 7.902666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1573
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.375,
+      "epoch": 2.5183999999999997,
+      "grad_norm": 0.008180846088264191,
+      "kl": 0.13525390625,
+      "learning_rate": 7.901333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1574
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.734375,
+      "epoch": 2.52,
+      "grad_norm": 0.007478267621127684,
+      "kl": 0.12255859375,
+      "learning_rate": 7.9e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1575
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.34375,
+      "epoch": 2.5216,
+      "grad_norm": 9.889973840518275,
+      "kl": 0.13134765625,
+      "learning_rate": 7.898666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.671875,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.671875,
+      "step": 1576
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.34375,
+      "epoch": 2.5232,
+      "grad_norm": 0.017696220277703487,
+      "kl": 0.12060546875,
+      "learning_rate": 7.897333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1577
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.265625,
+      "epoch": 2.5248,
+      "grad_norm": 0.007877113015754358,
+      "kl": 0.1318359375,
+      "learning_rate": 7.895999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1578
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.34375,
+      "epoch": 2.5263999999999998,
+      "grad_norm": 0.008201458478169152,
+      "kl": 0.1259765625,
+      "learning_rate": 7.894666666666666e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 1579
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.890625,
+      "epoch": 2.528,
+      "grad_norm": 13.39941969861168,
+      "kl": 0.116943359375,
+      "learning_rate": 7.893333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.6875,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.6875,
+      "step": 1580
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.625,
+      "epoch": 2.5296,
+      "grad_norm": 0.013734091552489154,
+      "kl": 0.13427734375,
+      "learning_rate": 7.892e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1581
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.578125,
+      "epoch": 2.5312,
+      "grad_norm": 12.46284176684884,
+      "kl": 0.130615234375,
+      "learning_rate": 7.890666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.46875,
+      "reward_std": 0.0883883461356163,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.46875,
+      "step": 1582
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.96875,
+      "epoch": 2.5328,
+      "grad_norm": 0.008481442353425758,
+      "kl": 0.130615234375,
+      "learning_rate": 7.889333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1583
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.96875,
+      "epoch": 2.5343999999999998,
+      "grad_norm": 0.006124327674472044,
+      "kl": 0.11572265625,
+      "learning_rate": 7.887999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1584
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.796875,
+      "epoch": 2.536,
+      "grad_norm": 0.01518917738077803,
+      "kl": 0.12890625,
+      "learning_rate": 7.886666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1585
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.046875,
+      "epoch": 2.5376,
+      "grad_norm": 0.008343522720730502,
+      "kl": 0.123779296875,
+      "learning_rate": 7.885333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1586
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.53125,
+      "epoch": 2.5392,
+      "grad_norm": 0.00728876775073092,
+      "kl": 0.121826171875,
+      "learning_rate": 7.883999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1587
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.171875,
+      "epoch": 2.5408,
+      "grad_norm": 0.00691174879044807,
+      "kl": 0.116943359375,
+      "learning_rate": 7.882666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1588
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.234375,
+      "epoch": 2.5423999999999998,
+      "grad_norm": 0.016557602364851248,
+      "kl": 0.134765625,
+      "learning_rate": 7.881333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1589
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.796875,
+      "epoch": 2.544,
+      "grad_norm": 0.007007904547742765,
+      "kl": 0.111572265625,
+      "learning_rate": 7.88e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 1590
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.28125,
+      "epoch": 2.5456,
+      "grad_norm": 0.009491751275883557,
+      "kl": 0.13427734375,
+      "learning_rate": 7.878666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1591
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.15625,
+      "epoch": 2.5472,
+      "grad_norm": 0.007146487487690065,
+      "kl": 0.121337890625,
+      "learning_rate": 7.877333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1592
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.78125,
+      "epoch": 2.5488,
+      "grad_norm": 0.007126196012222461,
+      "kl": 0.119140625,
+      "learning_rate": 7.875999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1593
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.734375,
+      "epoch": 2.5504,
+      "grad_norm": 8.230484471868994,
+      "kl": 0.126708984375,
+      "learning_rate": 7.874666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 1594
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.65625,
+      "epoch": 2.552,
+      "grad_norm": 0.0081563502357852,
+      "kl": 0.118896484375,
+      "learning_rate": 7.873333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1595
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.109375,
+      "epoch": 2.5536,
+      "grad_norm": 0.00823283904966398,
+      "kl": 0.13037109375,
+      "learning_rate": 7.872e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1596
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.703125,
+      "epoch": 2.5552,
+      "grad_norm": 0.009885422449945419,
+      "kl": 0.119384765625,
+      "learning_rate": 7.870666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1597
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.171875,
+      "epoch": 2.5568,
+      "grad_norm": 0.008795787377546759,
+      "kl": 0.121337890625,
+      "learning_rate": 7.869333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1598
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.25,
+      "epoch": 2.5584,
+      "grad_norm": 15.906554395038292,
+      "kl": 0.12109375,
+      "learning_rate": 7.868e-07,
+      "loss": 0.0001,
+      "reward": 1.71875,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.71875,
+      "step": 1599
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.03125,
+      "epoch": 2.56,
+      "grad_norm": 0.00759617336263763,
+      "kl": 0.123779296875,
+      "learning_rate": 7.866666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1600
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.375,
+      "epoch": 2.5616,
+      "grad_norm": 0.008926979314322073,
+      "kl": 0.11962890625,
+      "learning_rate": 7.865333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1601
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.578125,
+      "epoch": 2.5632,
+      "grad_norm": 0.006443619193664467,
+      "kl": 0.11083984375,
+      "learning_rate": 7.864e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1602
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.328125,
+      "epoch": 2.5648,
+      "grad_norm": 10.241608625605934,
+      "kl": 0.112548828125,
+      "learning_rate": 7.862666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.359375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.359375,
+      "step": 1603
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.25,
+      "epoch": 2.5664,
+      "grad_norm": 0.006785746527106827,
+      "kl": 0.10791015625,
+      "learning_rate": 7.861333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1604
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.78125,
+      "epoch": 2.568,
+      "grad_norm": 0.007097219511457193,
+      "kl": 0.109130859375,
+      "learning_rate": 7.86e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1605
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.328125,
+      "epoch": 2.5696,
+      "grad_norm": 0.05664926895033735,
+      "kl": 0.119873046875,
+      "learning_rate": 7.858666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1606
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.484375,
+      "epoch": 2.5712,
+      "grad_norm": 0.01096511076818798,
+      "kl": 0.130615234375,
+      "learning_rate": 7.857333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1607
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.0625,
+      "epoch": 2.5728,
+      "grad_norm": 0.021483104503776947,
+      "kl": 0.10888671875,
+      "learning_rate": 7.855999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1608
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.875,
+      "epoch": 2.5744,
+      "grad_norm": 0.009374614083399764,
+      "kl": 0.118896484375,
+      "learning_rate": 7.854666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1609
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.53125,
+      "epoch": 2.576,
+      "grad_norm": 0.009772647522807107,
+      "kl": 0.111572265625,
+      "learning_rate": 7.853333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1610
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.5625,
+      "epoch": 2.5776,
+      "grad_norm": 0.0915727782615796,
+      "kl": 0.11962890625,
+      "learning_rate": 7.852e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1611
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.59375,
+      "epoch": 2.5792,
+      "grad_norm": 0.006662429073930215,
+      "kl": 0.109375,
+      "learning_rate": 7.850666666666666e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 1612
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 143.859375,
+      "epoch": 2.5808,
+      "grad_norm": 0.007277514590620648,
+      "kl": 0.11181640625,
+      "learning_rate": 7.849333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1613
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.5,
+      "epoch": 2.5824,
+      "grad_norm": 0.0067172761670961775,
+      "kl": 0.107666015625,
+      "learning_rate": 7.848e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1614
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.84375,
+      "epoch": 2.584,
+      "grad_norm": 0.007007566012541276,
+      "kl": 0.10693359375,
+      "learning_rate": 7.846666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1615
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.65625,
+      "epoch": 2.5856,
+      "grad_norm": 2.528354367424871,
+      "kl": 0.10498046875,
+      "learning_rate": 7.845333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.984375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.984375,
+      "step": 1616
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.296875,
+      "epoch": 2.5872,
+      "grad_norm": 0.036437032013916126,
+      "kl": 0.105712890625,
+      "learning_rate": 7.844e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1617
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.28125,
+      "epoch": 2.5888,
+      "grad_norm": 0.007715149483937192,
+      "kl": 0.10986328125,
+      "learning_rate": 7.842666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1618
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.375,
+      "epoch": 2.5904,
+      "grad_norm": 0.021135584552658092,
+      "kl": 0.116943359375,
+      "learning_rate": 7.841333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1619
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.875,
+      "epoch": 2.592,
+      "grad_norm": 0.006741620560903608,
+      "kl": 0.11083984375,
+      "learning_rate": 7.84e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1620
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.421875,
+      "epoch": 2.5936,
+      "grad_norm": 0.00668945304121773,
+      "kl": 0.101806640625,
+      "learning_rate": 7.838666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1621
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 142.078125,
+      "epoch": 2.5952,
+      "grad_norm": 0.007334910008301085,
+      "kl": 0.11181640625,
+      "learning_rate": 7.837333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1622
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.203125,
+      "epoch": 2.5968,
+      "grad_norm": 0.006780614761817589,
+      "kl": 0.10498046875,
+      "learning_rate": 7.835999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1623
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.375,
+      "epoch": 2.5984,
+      "grad_norm": 0.014110826501394294,
+      "kl": 0.120849609375,
+      "learning_rate": 7.834666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1624
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.75,
+      "epoch": 2.6,
+      "grad_norm": 0.0064636473485198105,
+      "kl": 0.11279296875,
+      "learning_rate": 7.833333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1625
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.40625,
+      "epoch": 2.6016,
+      "grad_norm": 0.006905756734445843,
+      "kl": 0.10791015625,
+      "learning_rate": 7.832e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1626
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.515625,
+      "epoch": 2.6032,
+      "grad_norm": 6.940555170421396,
+      "kl": 0.113037109375,
+      "learning_rate": 7.830666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.8125,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.8125,
+      "step": 1627
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.1875,
+      "epoch": 2.6048,
+      "grad_norm": 29.532318803339027,
+      "kl": 0.109375,
+      "learning_rate": 7.829333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.203125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.203125,
+      "step": 1628
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 141.984375,
+      "epoch": 2.6064,
+      "grad_norm": 0.007797199982817574,
+      "kl": 0.109619140625,
+      "learning_rate": 7.828e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1629
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.265625,
+      "epoch": 2.608,
+      "grad_norm": 0.016894808945428424,
+      "kl": 0.109619140625,
+      "learning_rate": 7.826666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1630
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 143.234375,
+      "epoch": 2.6096,
+      "grad_norm": 6.094889336128218,
+      "kl": 0.103759765625,
+      "learning_rate": 7.825333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.6875,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.6875,
+      "step": 1631
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.53125,
+      "epoch": 2.6112,
+      "grad_norm": 0.006430818940382626,
+      "kl": 0.1103515625,
+      "learning_rate": 7.823999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1632
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.828125,
+      "epoch": 2.6128,
+      "grad_norm": 0.006498409481574618,
+      "kl": 0.113525390625,
+      "learning_rate": 7.822666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1633
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.40625,
+      "epoch": 2.6144,
+      "grad_norm": 0.007520967747004283,
+      "kl": 0.1044921875,
+      "learning_rate": 7.821333333333333e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 1634
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.5,
+      "epoch": 2.616,
+      "grad_norm": 0.006285016611299358,
+      "kl": 0.110107421875,
+      "learning_rate": 7.82e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1635
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.734375,
+      "epoch": 2.6176,
+      "grad_norm": 0.007574438835448996,
+      "kl": 0.113525390625,
+      "learning_rate": 7.818666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1636
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.484375,
+      "epoch": 2.6192,
+      "grad_norm": 0.007067190706230408,
+      "kl": 0.108154296875,
+      "learning_rate": 7.817333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1637
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 141.359375,
+      "epoch": 2.6208,
+      "grad_norm": 0.006124878885335162,
+      "kl": 0.112060546875,
+      "learning_rate": 7.816e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1638
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.65625,
+      "epoch": 2.6224,
+      "grad_norm": 0.015102492856587995,
+      "kl": 0.11474609375,
+      "learning_rate": 7.814666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1639
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.703125,
+      "epoch": 2.624,
+      "grad_norm": 0.006607009741009194,
+      "kl": 0.11474609375,
+      "learning_rate": 7.813333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1640
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 142.34375,
+      "epoch": 2.6256,
+      "grad_norm": 29.876819091134532,
+      "kl": 0.102294921875,
+      "learning_rate": 7.811999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.859375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.859375,
+      "step": 1641
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.1875,
+      "epoch": 2.6272,
+      "grad_norm": 0.009254752583174132,
+      "kl": 0.107421875,
+      "learning_rate": 7.810666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1642
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.5,
+      "epoch": 2.6288,
+      "grad_norm": 0.07525219201174065,
+      "kl": 0.115234375,
+      "learning_rate": 7.809333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1643
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.578125,
+      "epoch": 2.6304,
+      "grad_norm": 0.007983549367797615,
+      "kl": 0.103759765625,
+      "learning_rate": 7.808e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1644
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.34375,
+      "epoch": 2.632,
+      "grad_norm": 169.99125475113172,
+      "kl": 0.110595703125,
+      "learning_rate": 7.806666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.640625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.640625,
+      "step": 1645
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.9375,
+      "epoch": 2.6336,
+      "grad_norm": 13.396506652065192,
+      "kl": 0.11328125,
+      "learning_rate": 7.805333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.6875,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.6875,
+      "step": 1646
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.359375,
+      "epoch": 2.6352,
+      "grad_norm": 9.880946199644255,
+      "kl": 0.111572265625,
+      "learning_rate": 7.804e-07,
+      "loss": 0.0001,
+      "reward": 1.859375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.859375,
+      "step": 1647
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.234375,
+      "epoch": 2.6368,
+      "grad_norm": 0.007046198597124093,
+      "kl": 0.113037109375,
+      "learning_rate": 7.802666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1648
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.9375,
+      "epoch": 2.6384,
+      "grad_norm": 8.39215219584665,
+      "kl": 0.10595703125,
+      "learning_rate": 7.801333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.859375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.859375,
+      "step": 1649
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.0,
+      "epoch": 2.64,
+      "grad_norm": 0.007117195927571878,
+      "kl": 0.112060546875,
+      "learning_rate": 7.799999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1650
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.765625,
+      "epoch": 2.6416,
+      "grad_norm": 0.006379626976047427,
+      "kl": 0.099365234375,
+      "learning_rate": 7.798666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1651
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.515625,
+      "epoch": 2.6432,
+      "grad_norm": 0.010511343693236084,
+      "kl": 0.099853515625,
+      "learning_rate": 7.797333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1652
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 145.359375,
+      "epoch": 2.6448,
+      "grad_norm": 0.006621656022924609,
+      "kl": 0.115234375,
+      "learning_rate": 7.795999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1653
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.796875,
+      "epoch": 2.6464,
+      "grad_norm": 0.006081655533388503,
+      "kl": 0.106689453125,
+      "learning_rate": 7.794666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1654
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.0,
+      "epoch": 2.648,
+      "grad_norm": 0.013111160099017157,
+      "kl": 0.10498046875,
+      "learning_rate": 7.793333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1655
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 145.140625,
+      "epoch": 2.6496,
+      "grad_norm": 0.022550312021045005,
+      "kl": 0.105712890625,
+      "learning_rate": 7.792e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1656
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.453125,
+      "epoch": 2.6512000000000002,
+      "grad_norm": 0.006915808427939658,
+      "kl": 0.1064453125,
+      "learning_rate": 7.790666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1657
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.796875,
+      "epoch": 2.6528,
+      "grad_norm": 54.37700148617557,
+      "kl": 0.105712890625,
+      "learning_rate": 7.789333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.796875,
+      "reward_std": 0.11100947856903076,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.796875,
+      "step": 1658
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.640625,
+      "epoch": 2.6544,
+      "grad_norm": 0.007890393302185711,
+      "kl": 0.098388671875,
+      "learning_rate": 7.788000000000001e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1659
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.484375,
+      "epoch": 2.656,
+      "grad_norm": 15.566283095551515,
+      "kl": 0.106689453125,
+      "learning_rate": 7.786666666666665e-07,
+      "loss": 0.0001,
+      "reward": 1.890625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.890625,
+      "step": 1660
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.28125,
+      "epoch": 2.6576,
+      "grad_norm": 0.006930677083844174,
+      "kl": 0.102783203125,
+      "learning_rate": 7.785333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1661
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.875,
+      "epoch": 2.6592000000000002,
+      "grad_norm": 2.592224783058027,
+      "kl": 0.115478515625,
+      "learning_rate": 7.783999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.515625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.515625,
+      "step": 1662
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.8125,
+      "epoch": 2.6608,
+      "grad_norm": 0.007488860927936875,
+      "kl": 0.107421875,
+      "learning_rate": 7.782666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1663
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.8125,
+      "epoch": 2.6624,
+      "grad_norm": 0.011596756589339629,
+      "kl": 0.10107421875,
+      "learning_rate": 7.781333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1664
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.828125,
+      "epoch": 2.664,
+      "grad_norm": 0.019142228338803715,
+      "kl": 0.114990234375,
+      "learning_rate": 7.78e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1665
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.15625,
+      "epoch": 2.6656,
+      "grad_norm": 0.00783707965349344,
+      "kl": 0.10986328125,
+      "learning_rate": 7.778666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1666
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.71875,
+      "epoch": 2.6672000000000002,
+      "grad_norm": 0.006833131393325807,
+      "kl": 0.11279296875,
+      "learning_rate": 7.777333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1667
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.03125,
+      "epoch": 2.6688,
+      "grad_norm": 0.005793471814188504,
+      "kl": 0.10791015625,
+      "learning_rate": 7.776e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1668
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.546875,
+      "epoch": 2.6704,
+      "grad_norm": 0.01243231274149858,
+      "kl": 0.109375,
+      "learning_rate": 7.774666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1669
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.1875,
+      "epoch": 2.672,
+      "grad_norm": 1.7596916389773842,
+      "kl": 0.105712890625,
+      "learning_rate": 7.773333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.640625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.640625,
+      "step": 1670
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 141.46875,
+      "epoch": 2.6736,
+      "grad_norm": 0.017892580996176858,
+      "kl": 0.107666015625,
+      "learning_rate": 7.771999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1671
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.53125,
+      "epoch": 2.6752000000000002,
+      "grad_norm": 0.006265962000394917,
+      "kl": 0.104248046875,
+      "learning_rate": 7.770666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1672
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.015625,
+      "epoch": 2.6768,
+      "grad_norm": 8.703551662355471,
+      "kl": 0.101318359375,
+      "learning_rate": 7.769333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.921875,
+      "reward_std": 0.11100947856903076,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.921875,
+      "step": 1673
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.3125,
+      "epoch": 2.6784,
+      "grad_norm": 5.097500399932299,
+      "kl": 0.106689453125,
+      "learning_rate": 7.768e-07,
+      "loss": 0.0001,
+      "reward": 1.71875,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.71875,
+      "step": 1674
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.859375,
+      "epoch": 2.68,
+      "grad_norm": 21.863839327838857,
+      "kl": 0.126220703125,
+      "learning_rate": 7.766666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.796875,
+      "reward_std": 0.12255740165710449,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.796875,
+      "step": 1675
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.6875,
+      "epoch": 2.6816,
+      "grad_norm": 0.00843387786099691,
+      "kl": 0.10986328125,
+      "learning_rate": 7.765333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1676
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.84375,
+      "epoch": 2.6832000000000003,
+      "grad_norm": 0.007128288072893128,
+      "kl": 0.1083984375,
+      "learning_rate": 7.764e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1677
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.359375,
+      "epoch": 2.6848,
+      "grad_norm": 0.005977533810263941,
+      "kl": 0.105224609375,
+      "learning_rate": 7.762666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1678
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.46875,
+      "epoch": 2.6864,
+      "grad_norm": 0.00901277550035278,
+      "kl": 0.108642578125,
+      "learning_rate": 7.761333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1679
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.25,
+      "epoch": 2.6879999999999997,
+      "grad_norm": 0.0066610410431408125,
+      "kl": 0.103759765625,
+      "learning_rate": 7.76e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1680
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.546875,
+      "epoch": 2.6896,
+      "grad_norm": 0.00635371431436178,
+      "kl": 0.107177734375,
+      "learning_rate": 7.758666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1681
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 145.21875,
+      "epoch": 2.6912000000000003,
+      "grad_norm": 0.008042238813309241,
+      "kl": 0.103271484375,
+      "learning_rate": 7.757333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1682
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.25,
+      "epoch": 2.6928,
+      "grad_norm": 0.007034651998519269,
+      "kl": 0.10107421875,
+      "learning_rate": 7.755999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1683
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.703125,
+      "epoch": 2.6944,
+      "grad_norm": 0.006956181638360772,
+      "kl": 0.1064453125,
+      "learning_rate": 7.754666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1684
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.171875,
+      "epoch": 2.6959999999999997,
+      "grad_norm": 0.016391456848543463,
+      "kl": 0.103271484375,
+      "learning_rate": 7.753333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1685
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.03125,
+      "epoch": 2.6976,
+      "grad_norm": 0.005815304240063775,
+      "kl": 0.10791015625,
+      "learning_rate": 7.752e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1686
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.703125,
+      "epoch": 2.6992000000000003,
+      "grad_norm": 0.05174649471587082,
+      "kl": 0.107421875,
+      "learning_rate": 7.750666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1687
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 141.828125,
+      "epoch": 2.7008,
+      "grad_norm": 0.0052558531209378765,
+      "kl": 0.105224609375,
+      "learning_rate": 7.749333333333333e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 1688
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.234375,
+      "epoch": 2.7024,
+      "grad_norm": 8.158478064303543,
+      "kl": 0.10205078125,
+      "learning_rate": 7.748e-07,
+      "loss": 0.0001,
+      "reward": 1.515625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.515625,
+      "step": 1689
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.078125,
+      "epoch": 2.7039999999999997,
+      "grad_norm": 0.019018243978923273,
+      "kl": 0.11767578125,
+      "learning_rate": 7.746666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1690
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.40625,
+      "epoch": 2.7056,
+      "grad_norm": 0.0057027753987763385,
+      "kl": 0.10888671875,
+      "learning_rate": 7.745333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1691
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.203125,
+      "epoch": 2.7072000000000003,
+      "grad_norm": 15.08952392276304,
+      "kl": 0.11181640625,
+      "learning_rate": 7.743999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.8125,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.8125,
+      "step": 1692
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.421875,
+      "epoch": 2.7088,
+      "grad_norm": 0.008149551068969852,
+      "kl": 0.10546875,
+      "learning_rate": 7.742666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1693
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 142.984375,
+      "epoch": 2.7104,
+      "grad_norm": 10.388213140181138,
+      "kl": 0.11328125,
+      "learning_rate": 7.741333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.71875,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.71875,
+      "step": 1694
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 142.3125,
+      "epoch": 2.7119999999999997,
+      "grad_norm": 0.02382729814762338,
+      "kl": 0.115966796875,
+      "learning_rate": 7.74e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 1695
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.296875,
+      "epoch": 2.7136,
+      "grad_norm": 0.006184387582969016,
+      "kl": 0.107177734375,
+      "learning_rate": 7.738666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1696
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.078125,
+      "epoch": 2.7152,
+      "grad_norm": 83.36840895098304,
+      "kl": 0.13232421875,
+      "learning_rate": 7.737333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.703125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.703125,
+      "step": 1697
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.703125,
+      "epoch": 2.7168,
+      "grad_norm": 0.005861162190652896,
+      "kl": 0.1064453125,
+      "learning_rate": 7.735999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1698
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.515625,
+      "epoch": 2.7184,
+      "grad_norm": 0.012591626200442565,
+      "kl": 0.10888671875,
+      "learning_rate": 7.734666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1699
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.671875,
+      "epoch": 2.7199999999999998,
+      "grad_norm": 0.006448009938411066,
+      "kl": 0.109130859375,
+      "learning_rate": 7.733333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1700
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.65625,
+      "epoch": 2.7216,
+      "grad_norm": 5.237898356734469,
+      "kl": 0.09814453125,
+      "learning_rate": 7.732e-07,
+      "loss": 0.0001,
+      "reward": 1.609375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.609375,
+      "step": 1701
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.65625,
+      "epoch": 2.7232,
+      "grad_norm": 0.015344086962808468,
+      "kl": 0.10595703125,
+      "learning_rate": 7.730666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1702
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 145.3125,
+      "epoch": 2.7248,
+      "grad_norm": 0.007221737831532145,
+      "kl": 0.10205078125,
+      "learning_rate": 7.729333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1703
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 142.0625,
+      "epoch": 2.7264,
+      "grad_norm": 0.010745484986598665,
+      "kl": 0.10595703125,
+      "learning_rate": 7.728e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 1704
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 141.96875,
+      "epoch": 2.7279999999999998,
+      "grad_norm": 0.00636096719436834,
+      "kl": 0.1015625,
+      "learning_rate": 7.726666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1705
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 143.265625,
+      "epoch": 2.7296,
+      "grad_norm": 0.017317186641435636,
+      "kl": 0.104248046875,
+      "learning_rate": 7.725333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1706
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 144.921875,
+      "epoch": 2.7312,
+      "grad_norm": 0.0075232113322165756,
+      "kl": 0.104248046875,
+      "learning_rate": 7.723999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1707
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.421875,
+      "epoch": 2.7328,
+      "grad_norm": 0.011535777613743017,
+      "kl": 0.10693359375,
+      "learning_rate": 7.722666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1708
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.875,
+      "epoch": 2.7344,
+      "grad_norm": 0.01169127301993601,
+      "kl": 0.108642578125,
+      "learning_rate": 7.721333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1709
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 145.078125,
+      "epoch": 2.7359999999999998,
+      "grad_norm": 0.007915621291706707,
+      "kl": 0.099365234375,
+      "learning_rate": 7.72e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1710
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 144.0625,
+      "epoch": 2.7376,
+      "grad_norm": 0.005653467292016955,
+      "kl": 0.101318359375,
+      "learning_rate": 7.718666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1711
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 141.859375,
+      "epoch": 2.7392,
+      "grad_norm": 0.005274600093969899,
+      "kl": 0.103759765625,
+      "learning_rate": 7.717333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1712
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.65625,
+      "epoch": 2.7408,
+      "grad_norm": 0.007740784137181854,
+      "kl": 0.109375,
+      "learning_rate": 7.716e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1713
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 143.859375,
+      "epoch": 2.7424,
+      "grad_norm": 8.818275462163852,
+      "kl": 0.0927734375,
+      "learning_rate": 7.714666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.859375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.859375,
+      "step": 1714
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.0,
+      "epoch": 2.7439999999999998,
+      "grad_norm": 0.007062306564056496,
+      "kl": 0.099853515625,
+      "learning_rate": 7.713333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1715
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 149.9375,
+      "epoch": 2.7456,
+      "grad_norm": 0.008022107717079634,
+      "kl": 0.1015625,
+      "learning_rate": 7.711999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1716
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 145.28125,
+      "epoch": 2.7472,
+      "grad_norm": 0.00816640020088649,
+      "kl": 0.096923828125,
+      "learning_rate": 7.710666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1717
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.9375,
+      "epoch": 2.7488,
+      "grad_norm": 0.012199662323228241,
+      "kl": 0.1171875,
+      "learning_rate": 7.709333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1718
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 146.078125,
+      "epoch": 2.7504,
+      "grad_norm": 0.006486233535802742,
+      "kl": 0.0986328125,
+      "learning_rate": 7.708e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1719
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.484375,
+      "epoch": 2.752,
+      "grad_norm": 0.007905956754987077,
+      "kl": 0.103515625,
+      "learning_rate": 7.706666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1720
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.4375,
+      "epoch": 2.7536,
+      "grad_norm": 1.2385738015273997,
+      "kl": 0.104248046875,
+      "learning_rate": 7.705333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.453125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.453125,
+      "step": 1721
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 148.59375,
+      "epoch": 2.7552,
+      "grad_norm": 0.007380331586295271,
+      "kl": 0.10205078125,
+      "learning_rate": 7.704e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1722
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.8125,
+      "epoch": 2.7568,
+      "grad_norm": 0.010973480256519964,
+      "kl": 0.10693359375,
+      "learning_rate": 7.702666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1723
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 154.375,
+      "epoch": 2.7584,
+      "grad_norm": 0.008919950161917711,
+      "kl": 0.10693359375,
+      "learning_rate": 7.701333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1724
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 159.515625,
+      "epoch": 2.76,
+      "grad_norm": 0.007961703187107419,
+      "kl": 0.10400390625,
+      "learning_rate": 7.699999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1725
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 155.5625,
+      "epoch": 2.7616,
+      "grad_norm": 0.006236705754990647,
+      "kl": 0.110107421875,
+      "learning_rate": 7.698666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1726
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 160.0625,
+      "epoch": 2.7632,
+      "grad_norm": 0.01205363441407681,
+      "kl": 0.10205078125,
+      "learning_rate": 7.697333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1727
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 157.296875,
+      "epoch": 2.7648,
+      "grad_norm": 5.132030826871741,
+      "kl": 0.109130859375,
+      "learning_rate": 7.695999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.53125,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.53125,
+      "step": 1728
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 161.71875,
+      "epoch": 2.7664,
+      "grad_norm": 0.007074312184269987,
+      "kl": 0.10595703125,
+      "learning_rate": 7.694666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1729
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 158.703125,
+      "epoch": 2.768,
+      "grad_norm": 0.009504573625913287,
+      "kl": 0.11181640625,
+      "learning_rate": 7.693333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1730
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 160.46875,
+      "epoch": 2.7696,
+      "grad_norm": 0.007550525408474002,
+      "kl": 0.1025390625,
+      "learning_rate": 7.692e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1731
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 164.90625,
+      "epoch": 2.7712,
+      "grad_norm": 4.494506116650903,
+      "kl": 0.1123046875,
+      "learning_rate": 7.690666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.984375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.984375,
+      "step": 1732
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 169.34375,
+      "epoch": 2.7728,
+      "grad_norm": 0.00674324900205177,
+      "kl": 0.11279296875,
+      "learning_rate": 7.689333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1733
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 157.34375,
+      "epoch": 2.7744,
+      "grad_norm": 0.8732633554722592,
+      "kl": 0.11083984375,
+      "learning_rate": 7.688000000000001e-07,
+      "loss": 0.0001,
+      "reward": 1.515625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.515625,
+      "step": 1734
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 169.90625,
+      "epoch": 2.776,
+      "grad_norm": 0.0061152432738147185,
+      "kl": 0.1083984375,
+      "learning_rate": 7.686666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1735
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 165.328125,
+      "epoch": 2.7776,
+      "grad_norm": 0.017241393690673765,
+      "kl": 0.10595703125,
+      "learning_rate": 7.685333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1736
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 176.65625,
+      "epoch": 2.7792,
+      "grad_norm": 1.0598848172361681,
+      "kl": 0.110107421875,
+      "learning_rate": 7.683999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.75,
+      "step": 1737
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 171.859375,
+      "epoch": 2.7808,
+      "grad_norm": 0.006820931223843641,
+      "kl": 0.11181640625,
+      "learning_rate": 7.682666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1738
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 180.171875,
+      "epoch": 2.7824,
+      "grad_norm": 1.2164980308922482,
+      "kl": 0.107177734375,
+      "learning_rate": 7.681333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.59375,
+      "reward_std": 0.0883883461356163,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.609375,
+      "step": 1739
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 171.28125,
+      "epoch": 2.784,
+      "grad_norm": 0.02364912138183742,
+      "kl": 0.11083984375,
+      "learning_rate": 7.68e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1740
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 176.453125,
+      "epoch": 2.7856,
+      "grad_norm": 0.006484967449104372,
+      "kl": 0.11572265625,
+      "learning_rate": 7.678666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1741
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 164.140625,
+      "epoch": 2.7872,
+      "grad_norm": 0.00887068549390288,
+      "kl": 0.116943359375,
+      "learning_rate": 7.677333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.25,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.25,
+      "step": 1742
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 159.0,
+      "epoch": 2.7888,
+      "grad_norm": 0.008102216730999647,
+      "kl": 0.120361328125,
+      "learning_rate": 7.676e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1743
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 166.8125,
+      "epoch": 2.7904,
+      "grad_norm": 0.007782452254656654,
+      "kl": 0.124755859375,
+      "learning_rate": 7.674666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1744
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 158.421875,
+      "epoch": 2.792,
+      "grad_norm": 0.010463020027306901,
+      "kl": 0.12646484375,
+      "learning_rate": 7.673333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1745
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 154.703125,
+      "epoch": 2.7936,
+      "grad_norm": 6.640592628236894,
+      "kl": 0.1259765625,
+      "learning_rate": 7.671999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.703125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.703125,
+      "step": 1746
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 159.578125,
+      "epoch": 2.7952,
+      "grad_norm": 0.007783933597533558,
+      "kl": 0.128662109375,
+      "learning_rate": 7.670666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1747
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 159.21875,
+      "epoch": 2.7968,
+      "grad_norm": 0.00828924721882879,
+      "kl": 0.13134765625,
+      "learning_rate": 7.669333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1748
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 153.59375,
+      "epoch": 2.7984,
+      "grad_norm": 0.007261762985949499,
+      "kl": 0.1220703125,
+      "learning_rate": 7.668e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1749
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 144.265625,
+      "epoch": 2.8,
+      "grad_norm": 0.009560720357756068,
+      "kl": 0.1416015625,
+      "learning_rate": 7.666666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1750
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 145.921875,
+      "epoch": 2.8016,
+      "grad_norm": 0.007106936839747563,
+      "kl": 0.13037109375,
+      "learning_rate": 7.665333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1751
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 153.9375,
+      "epoch": 2.8032,
+      "grad_norm": 0.9346227340418413,
+      "kl": 0.1357421875,
+      "learning_rate": 7.664e-07,
+      "loss": 0.0001,
+      "reward": 1.84375,
+      "reward_std": 0.0883883461356163,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.859375,
+      "step": 1752
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 148.796875,
+      "epoch": 2.8048,
+      "grad_norm": 0.007790220383188229,
+      "kl": 0.12890625,
+      "learning_rate": 7.662666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1753
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 146.328125,
+      "epoch": 2.8064,
+      "grad_norm": 0.014278928405413154,
+      "kl": 0.14599609375,
+      "learning_rate": 7.661333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1754
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 146.921875,
+      "epoch": 2.808,
+      "grad_norm": 0.01140967668124277,
+      "kl": 0.13720703125,
+      "learning_rate": 7.66e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1755
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 142.1875,
+      "epoch": 2.8096,
+      "grad_norm": 6.687581655164236,
+      "kl": 0.13916015625,
+      "learning_rate": 7.658666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.703125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.703125,
+      "step": 1756
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 146.59375,
+      "epoch": 2.8112,
+      "grad_norm": 8.746598488032802,
+      "kl": 0.17236328125,
+      "learning_rate": 7.657333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.53125,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.53125,
+      "step": 1757
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.1875,
+      "epoch": 2.8128,
+      "grad_norm": 17.15860921036231,
+      "kl": 0.14453125,
+      "learning_rate": 7.655999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.828125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.828125,
+      "step": 1758
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.328125,
+      "epoch": 2.8144,
+      "grad_norm": 0.013321922176924812,
+      "kl": 0.14990234375,
+      "learning_rate": 7.654666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1759
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.0625,
+      "epoch": 2.816,
+      "grad_norm": 0.009596032321532659,
+      "kl": 0.146484375,
+      "learning_rate": 7.653333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1760
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.875,
+      "epoch": 2.8176,
+      "grad_norm": 0.009130426898236102,
+      "kl": 0.14990234375,
+      "learning_rate": 7.652e-07,
+      "loss": 0.0002,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1761
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.953125,
+      "epoch": 2.8192,
+      "grad_norm": 0.008937358667235376,
+      "kl": 0.1572265625,
+      "learning_rate": 7.650666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1762
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.109375,
+      "epoch": 2.8208,
+      "grad_norm": 1.5938693639569004,
+      "kl": 0.1484375,
+      "learning_rate": 7.649333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.640625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.640625,
+      "step": 1763
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.328125,
+      "epoch": 2.8224,
+      "grad_norm": 0.018755651254053818,
+      "kl": 0.1640625,
+      "learning_rate": 7.648e-07,
+      "loss": 0.0002,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1764
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.421875,
+      "epoch": 2.824,
+      "grad_norm": 0.010572380884804718,
+      "kl": 0.1728515625,
+      "learning_rate": 7.646666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1765
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.953125,
+      "epoch": 2.8256,
+      "grad_norm": 0.015132882998692814,
+      "kl": 0.15087890625,
+      "learning_rate": 7.645333333333332e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1766
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.109375,
+      "epoch": 2.8272,
+      "grad_norm": 0.01001625966181196,
+      "kl": 0.177734375,
+      "learning_rate": 7.643999999999999e-07,
+      "loss": 0.0002,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 1767
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.671875,
+      "epoch": 2.8288,
+      "grad_norm": 0.011927319703297662,
+      "kl": 0.16015625,
+      "learning_rate": 7.642666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1768
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.359375,
+      "epoch": 2.8304,
+      "grad_norm": 0.008724783901049577,
+      "kl": 0.15966796875,
+      "learning_rate": 7.641333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1769
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.671875,
+      "epoch": 2.832,
+      "grad_norm": 0.014088044016148674,
+      "kl": 0.1591796875,
+      "learning_rate": 7.64e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1770
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.203125,
+      "epoch": 2.8336,
+      "grad_norm": 0.03715738731079641,
+      "kl": 0.1640625,
+      "learning_rate": 7.638666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1771
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.921875,
+      "epoch": 2.8352,
+      "grad_norm": 0.014921118893095314,
+      "kl": 0.18505859375,
+      "learning_rate": 7.637333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.125,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.125,
+      "step": 1772
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.046875,
+      "epoch": 2.8368,
+      "grad_norm": 4.8291304385934914,
+      "kl": 0.1611328125,
+      "learning_rate": 7.635999999999999e-07,
+      "loss": 0.0002,
+      "reward": 1.484375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.484375,
+      "step": 1773
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.703125,
+      "epoch": 2.8384,
+      "grad_norm": 0.009814815953157249,
+      "kl": 0.17431640625,
+      "learning_rate": 7.634666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1774
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.515625,
+      "epoch": 2.84,
+      "grad_norm": 0.01127949355354609,
+      "kl": 0.16455078125,
+      "learning_rate": 7.633333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1775
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.375,
+      "epoch": 2.8416,
+      "grad_norm": 0.015628805651430364,
+      "kl": 0.18017578125,
+      "learning_rate": 7.632e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1776
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.9375,
+      "epoch": 2.8432,
+      "grad_norm": 2.3302702523759775,
+      "kl": 0.16015625,
+      "learning_rate": 7.630666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 1777
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.03125,
+      "epoch": 2.8448,
+      "grad_norm": 0.017823723204062523,
+      "kl": 0.16162109375,
+      "learning_rate": 7.629333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1778
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.703125,
+      "epoch": 2.8464,
+      "grad_norm": 0.00870576087108583,
+      "kl": 0.1767578125,
+      "learning_rate": 7.628e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1779
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.328125,
+      "epoch": 2.848,
+      "grad_norm": 12.620458228534641,
+      "kl": 0.15478515625,
+      "learning_rate": 7.626666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.609375,
+      "reward_std": 0.10205793380737305,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.609375,
+      "step": 1780
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.78125,
+      "epoch": 2.8496,
+      "grad_norm": 0.01091204229879105,
+      "kl": 0.1669921875,
+      "learning_rate": 7.625333333333332e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1781
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.40625,
+      "epoch": 2.8512,
+      "grad_norm": 1.0487403769459167,
+      "kl": 0.15966796875,
+      "learning_rate": 7.623999999999999e-07,
+      "loss": 0.0002,
+      "reward": 1.609375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.609375,
+      "step": 1782
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.5,
+      "epoch": 2.8528000000000002,
+      "grad_norm": 0.00939972680512294,
+      "kl": 0.154296875,
+      "learning_rate": 7.622666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1783
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.03125,
+      "epoch": 2.8544,
+      "grad_norm": 0.009991003118604137,
+      "kl": 0.1689453125,
+      "learning_rate": 7.621333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1784
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.484375,
+      "epoch": 2.856,
+      "grad_norm": 0.012753007651929736,
+      "kl": 0.1708984375,
+      "learning_rate": 7.62e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1785
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.078125,
+      "epoch": 2.8576,
+      "grad_norm": 0.05052540374355364,
+      "kl": 0.15673828125,
+      "learning_rate": 7.618666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1786
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.625,
+      "epoch": 2.8592,
+      "grad_norm": 0.014150502938216318,
+      "kl": 0.1533203125,
+      "learning_rate": 7.617333333333334e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1787
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.171875,
+      "epoch": 2.8608000000000002,
+      "grad_norm": 0.03176239768057956,
+      "kl": 0.1611328125,
+      "learning_rate": 7.616e-07,
+      "loss": 0.0002,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1788
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.671875,
+      "epoch": 2.8624,
+      "grad_norm": 0.014441186285909895,
+      "kl": 0.162109375,
+      "learning_rate": 7.614666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1789
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.140625,
+      "epoch": 2.864,
+      "grad_norm": 0.010934946221431919,
+      "kl": 0.15966796875,
+      "learning_rate": 7.613333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1790
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.0625,
+      "epoch": 2.8656,
+      "grad_norm": 0.013549389456323587,
+      "kl": 0.16455078125,
+      "learning_rate": 7.611999999999999e-07,
+      "loss": 0.0002,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1791
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.5,
+      "epoch": 2.8672,
+      "grad_norm": 0.009734151496723673,
+      "kl": 0.15966796875,
+      "learning_rate": 7.610666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1792
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.03125,
+      "epoch": 2.8688000000000002,
+      "grad_norm": 0.03861916769349973,
+      "kl": 0.181640625,
+      "learning_rate": 7.609333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1793
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.71875,
+      "epoch": 2.8704,
+      "grad_norm": 0.007941800802828008,
+      "kl": 0.15869140625,
+      "learning_rate": 7.608e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1794
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.6875,
+      "epoch": 2.872,
+      "grad_norm": 5.436484904758751,
+      "kl": 0.15625,
+      "learning_rate": 7.606666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.765625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.765625,
+      "step": 1795
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.5,
+      "epoch": 2.8736,
+      "grad_norm": 5.466734401031771,
+      "kl": 0.16650390625,
+      "learning_rate": 7.605333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.6875,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.6875,
+      "step": 1796
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.40625,
+      "epoch": 2.8752,
+      "grad_norm": 0.008085519841326735,
+      "kl": 0.150390625,
+      "learning_rate": 7.604e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1797
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.5,
+      "epoch": 2.8768000000000002,
+      "grad_norm": 0.0238486832892118,
+      "kl": 0.154296875,
+      "learning_rate": 7.602666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1798
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.21875,
+      "epoch": 2.8784,
+      "grad_norm": 0.011884077737762842,
+      "kl": 0.146484375,
+      "learning_rate": 7.601333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1799
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.140625,
+      "epoch": 2.88,
+      "grad_norm": 0.008978917475729735,
+      "kl": 0.138671875,
+      "learning_rate": 7.599999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1800
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.375,
+      "epoch": 2.8816,
+      "grad_norm": 0.008075030738199181,
+      "kl": 0.15087890625,
+      "learning_rate": 7.598666666666666e-07,
+      "loss": 0.0002,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 1801
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.46875,
+      "epoch": 2.8832,
+      "grad_norm": 0.0092184391026273,
+      "kl": 0.1474609375,
+      "learning_rate": 7.597333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1802
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.890625,
+      "epoch": 2.8848000000000003,
+      "grad_norm": 0.008764627841850312,
+      "kl": 0.15966796875,
+      "learning_rate": 7.596e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1803
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.421875,
+      "epoch": 2.8864,
+      "grad_norm": 0.009264973086308815,
+      "kl": 0.14599609375,
+      "learning_rate": 7.594666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1804
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.359375,
+      "epoch": 2.888,
+      "grad_norm": 11.474608218222391,
+      "kl": 0.1455078125,
+      "learning_rate": 7.593333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.359375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.359375,
+      "step": 1805
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.390625,
+      "epoch": 2.8895999999999997,
+      "grad_norm": 2.389199494205096,
+      "kl": 0.158203125,
+      "learning_rate": 7.592e-07,
+      "loss": 0.0002,
+      "reward": 1.890625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.890625,
+      "step": 1806
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.65625,
+      "epoch": 2.8912,
+      "grad_norm": 0.016975462572062797,
+      "kl": 0.14404296875,
+      "learning_rate": 7.590666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1807
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.5,
+      "epoch": 2.8928000000000003,
+      "grad_norm": 0.01485547963659778,
+      "kl": 0.14501953125,
+      "learning_rate": 7.589333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1808
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.265625,
+      "epoch": 2.8944,
+      "grad_norm": 0.010813106314057885,
+      "kl": 0.1533203125,
+      "learning_rate": 7.588e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1809
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.296875,
+      "epoch": 2.896,
+      "grad_norm": 9.935022412379652,
+      "kl": 0.13671875,
+      "learning_rate": 7.586666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.828125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.828125,
+      "step": 1810
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.65625,
+      "epoch": 2.8975999999999997,
+      "grad_norm": 7.794500284430041,
+      "kl": 0.14990234375,
+      "learning_rate": 7.585333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.484375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.484375,
+      "step": 1811
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.265625,
+      "epoch": 2.8992,
+      "grad_norm": 0.008140686722673008,
+      "kl": 0.14013671875,
+      "learning_rate": 7.583999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1812
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.390625,
+      "epoch": 2.9008000000000003,
+      "grad_norm": 0.009276974404094142,
+      "kl": 0.14892578125,
+      "learning_rate": 7.582666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1813
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.703125,
+      "epoch": 2.9024,
+      "grad_norm": 0.010810934231527921,
+      "kl": 0.14892578125,
+      "learning_rate": 7.581333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1814
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.78125,
+      "epoch": 2.904,
+      "grad_norm": 17.1310938107891,
+      "kl": 0.14697265625,
+      "learning_rate": 7.58e-07,
+      "loss": 0.0001,
+      "reward": 1.515625,
+      "reward_std": 0.10205793008208275,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.515625,
+      "step": 1815
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.65625,
+      "epoch": 2.9055999999999997,
+      "grad_norm": 0.018254678922458717,
+      "kl": 0.14453125,
+      "learning_rate": 7.578666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1816
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.140625,
+      "epoch": 2.9072,
+      "grad_norm": 5.666824117248223,
+      "kl": 0.1533203125,
+      "learning_rate": 7.577333333333334e-07,
+      "loss": 0.0002,
+      "reward": 1.640625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.640625,
+      "step": 1817
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.9375,
+      "epoch": 2.9088000000000003,
+      "grad_norm": 0.02469186402084686,
+      "kl": 0.14697265625,
+      "learning_rate": 7.576000000000001e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1818
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.921875,
+      "epoch": 2.9104,
+      "grad_norm": 0.018192012593657766,
+      "kl": 0.14453125,
+      "learning_rate": 7.574666666666665e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1819
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.90625,
+      "epoch": 2.912,
+      "grad_norm": 0.009611013155519335,
+      "kl": 0.14453125,
+      "learning_rate": 7.573333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1820
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.484375,
+      "epoch": 2.9135999999999997,
+      "grad_norm": 0.00912357227901762,
+      "kl": 0.130859375,
+      "learning_rate": 7.571999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1821
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.34375,
+      "epoch": 2.9152,
+      "grad_norm": 0.009374222803138262,
+      "kl": 0.14404296875,
+      "learning_rate": 7.570666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1822
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.640625,
+      "epoch": 2.9168,
+      "grad_norm": 0.008954553686005187,
+      "kl": 0.15185546875,
+      "learning_rate": 7.569333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1823
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.890625,
+      "epoch": 2.9184,
+      "grad_norm": 0.011311190083471976,
+      "kl": 0.142578125,
+      "learning_rate": 7.568e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1824
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.046875,
+      "epoch": 2.92,
+      "grad_norm": 0.008745790014859046,
+      "kl": 0.140625,
+      "learning_rate": 7.566666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1825
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.25,
+      "epoch": 2.9215999999999998,
+      "grad_norm": 0.0119032591952989,
+      "kl": 0.150390625,
+      "learning_rate": 7.565333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1826
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.234375,
+      "epoch": 2.9232,
+      "grad_norm": 0.009692099504814538,
+      "kl": 0.14111328125,
+      "learning_rate": 7.564e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1827
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.6875,
+      "epoch": 2.9248,
+      "grad_norm": 0.017727485377420894,
+      "kl": 0.14013671875,
+      "learning_rate": 7.562666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1828
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.609375,
+      "epoch": 2.9264,
+      "grad_norm": 0.007473922541645224,
+      "kl": 0.1298828125,
+      "learning_rate": 7.561333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1829
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.265625,
+      "epoch": 2.928,
+      "grad_norm": 0.007546379131989954,
+      "kl": 0.13525390625,
+      "learning_rate": 7.559999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1830
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.703125,
+      "epoch": 2.9295999999999998,
+      "grad_norm": 0.007664181072872164,
+      "kl": 0.13037109375,
+      "learning_rate": 7.558666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1831
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.703125,
+      "epoch": 2.9312,
+      "grad_norm": 0.013949503859706364,
+      "kl": 0.1416015625,
+      "learning_rate": 7.557333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1832
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.859375,
+      "epoch": 2.9328,
+      "grad_norm": 4.20579117523407,
+      "kl": 0.13671875,
+      "learning_rate": 7.556e-07,
+      "loss": 0.0001,
+      "reward": 1.984375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.984375,
+      "step": 1833
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.015625,
+      "epoch": 2.9344,
+      "grad_norm": 0.008522793312362009,
+      "kl": 0.12939453125,
+      "learning_rate": 7.554666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1834
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.4375,
+      "epoch": 2.936,
+      "grad_norm": 31.41555151402324,
+      "kl": 0.12841796875,
+      "learning_rate": 7.553333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.65625,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.65625,
+      "step": 1835
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.046875,
+      "epoch": 2.9375999999999998,
+      "grad_norm": 81.36030560025472,
+      "kl": 0.1455078125,
+      "learning_rate": 7.552e-07,
+      "loss": 0.0001,
+      "reward": 1.4375,
+      "reward_std": 0.10888782143592834,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.4375,
+      "step": 1836
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.03125,
+      "epoch": 2.9392,
+      "grad_norm": 18.165627545794347,
+      "kl": 0.1318359375,
+      "learning_rate": 7.550666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.421875,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.421875,
+      "step": 1837
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.671875,
+      "epoch": 2.9408,
+      "grad_norm": 6.530855655226578,
+      "kl": 0.14208984375,
+      "learning_rate": 7.549333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.4375,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.4375,
+      "step": 1838
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.640625,
+      "epoch": 2.9424,
+      "grad_norm": 0.012624181456619525,
+      "kl": 0.12841796875,
+      "learning_rate": 7.548e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1839
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.6875,
+      "epoch": 2.944,
+      "grad_norm": 0.007765166947503425,
+      "kl": 0.126220703125,
+      "learning_rate": 7.546666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1840
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.796875,
+      "epoch": 2.9455999999999998,
+      "grad_norm": 0.007340346911323464,
+      "kl": 0.138671875,
+      "learning_rate": 7.545333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1841
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.53125,
+      "epoch": 2.9472,
+      "grad_norm": 0.007357385918014669,
+      "kl": 0.134765625,
+      "learning_rate": 7.543999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1842
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.078125,
+      "epoch": 2.9488,
+      "grad_norm": 19.043927571340024,
+      "kl": 0.125732421875,
+      "learning_rate": 7.542666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.8125,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.8125,
+      "step": 1843
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.09375,
+      "epoch": 2.9504,
+      "grad_norm": 0.009941520396712998,
+      "kl": 0.129638671875,
+      "learning_rate": 7.541333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1844
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.703125,
+      "epoch": 2.952,
+      "grad_norm": 0.007782188660732586,
+      "kl": 0.12744140625,
+      "learning_rate": 7.54e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1845
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.90625,
+      "epoch": 2.9536,
+      "grad_norm": 0.009443268367075177,
+      "kl": 0.15087890625,
+      "learning_rate": 7.538666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1846
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.8125,
+      "epoch": 2.9552,
+      "grad_norm": 0.009774600201033327,
+      "kl": 0.13427734375,
+      "learning_rate": 7.537333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1847
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.984375,
+      "epoch": 2.9568,
+      "grad_norm": 0.007724143555965162,
+      "kl": 0.14013671875,
+      "learning_rate": 7.536e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1848
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.109375,
+      "epoch": 2.9584,
+      "grad_norm": 0.01178668339759549,
+      "kl": 0.138671875,
+      "learning_rate": 7.534666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1849
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.375,
+      "epoch": 2.96,
+      "grad_norm": 0.012428410074317877,
+      "kl": 0.13818359375,
+      "learning_rate": 7.533333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1850
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.71875,
+      "epoch": 2.9616,
+      "grad_norm": 0.04191284882671811,
+      "kl": 0.12646484375,
+      "learning_rate": 7.531999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1851
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.84375,
+      "epoch": 2.9632,
+      "grad_norm": 4.466433391989065,
+      "kl": 0.12841796875,
+      "learning_rate": 7.530666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.640625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.640625,
+      "step": 1852
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.296875,
+      "epoch": 2.9648,
+      "grad_norm": 0.006988325641410561,
+      "kl": 0.134033203125,
+      "learning_rate": 7.529333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1853
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.15625,
+      "epoch": 2.9664,
+      "grad_norm": 144.44014187740794,
+      "kl": 0.1376953125,
+      "learning_rate": 7.528e-07,
+      "loss": 0.0001,
+      "reward": 1.796875,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.796875,
+      "step": 1854
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.296875,
+      "epoch": 2.968,
+      "grad_norm": 0.007407822118918072,
+      "kl": 0.1279296875,
+      "learning_rate": 7.526666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1855
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.75,
+      "epoch": 2.9696,
+      "grad_norm": 0.007055622496078046,
+      "kl": 0.13232421875,
+      "learning_rate": 7.525333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1856
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.609375,
+      "epoch": 2.9712,
+      "grad_norm": 0.0074489513554066135,
+      "kl": 0.12841796875,
+      "learning_rate": 7.523999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1857
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.171875,
+      "epoch": 2.9728,
+      "grad_norm": 0.0061138306225497294,
+      "kl": 0.12744140625,
+      "learning_rate": 7.522666666666666e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 1858
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.140625,
+      "epoch": 2.9744,
+      "grad_norm": 0.007945615189364891,
+      "kl": 0.146484375,
+      "learning_rate": 7.521333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1859
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.0,
+      "epoch": 2.976,
+      "grad_norm": 22.332860040509885,
+      "kl": 0.12890625,
+      "learning_rate": 7.52e-07,
+      "loss": 0.0001,
+      "reward": 1.890625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.890625,
+      "step": 1860
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.296875,
+      "epoch": 2.9776,
+      "grad_norm": 19.96368932936411,
+      "kl": 0.13916015625,
+      "learning_rate": 7.518666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.953125,
+      "reward_std": 0.10205793008208275,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.953125,
+      "step": 1861
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.515625,
+      "epoch": 2.9792,
+      "grad_norm": 0.03415308291076324,
+      "kl": 0.19140625,
+      "learning_rate": 7.517333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1862
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.421875,
+      "epoch": 2.9808,
+      "grad_norm": 0.010459223319480732,
+      "kl": 0.13623046875,
+      "learning_rate": 7.516e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1863
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.90625,
+      "epoch": 2.9824,
+      "grad_norm": 26.833412366519187,
+      "kl": 0.150390625,
+      "learning_rate": 7.514666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.71875,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.71875,
+      "step": 1864
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.640625,
+      "epoch": 2.984,
+      "grad_norm": 0.007089239614694216,
+      "kl": 0.13134765625,
+      "learning_rate": 7.513333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1865
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.375,
+      "epoch": 2.9856,
+      "grad_norm": 0.007374581641835435,
+      "kl": 0.1455078125,
+      "learning_rate": 7.511999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1866
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.0625,
+      "epoch": 2.9872,
+      "grad_norm": 0.007886809035471173,
+      "kl": 0.1416015625,
+      "learning_rate": 7.510666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1867
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.0,
+      "epoch": 2.9888,
+      "grad_norm": 0.010858025259103734,
+      "kl": 0.14306640625,
+      "learning_rate": 7.509333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1868
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.28125,
+      "epoch": 2.9904,
+      "grad_norm": 0.011509381416080322,
+      "kl": 0.13427734375,
+      "learning_rate": 7.508e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1869
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.703125,
+      "epoch": 2.992,
+      "grad_norm": 0.009049539877442381,
+      "kl": 0.150390625,
+      "learning_rate": 7.506666666666667e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1870
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.734375,
+      "epoch": 2.9936,
+      "grad_norm": 0.013355960292918108,
+      "kl": 0.15234375,
+      "learning_rate": 7.505333333333334e-07,
+      "loss": 0.0002,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1871
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.078125,
+      "epoch": 2.9952,
+      "grad_norm": 0.011966805825640114,
+      "kl": 0.14697265625,
+      "learning_rate": 7.503999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1872
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.21875,
+      "epoch": 2.9968,
+      "grad_norm": 0.012642374676819099,
+      "kl": 0.13037109375,
+      "learning_rate": 7.502666666666666e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 1873
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.703125,
+      "epoch": 2.9984,
+      "grad_norm": 0.007816137874584846,
+      "kl": 0.13525390625,
+      "learning_rate": 7.501333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1874
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 102.875,
+      "epoch": 3.0,
+      "grad_norm": 0.009140219969111523,
+      "kl": 0.13037109375,
+      "learning_rate": 7.5e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1875
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.203125,
+      "epoch": 3.0016,
+      "grad_norm": 13.730869797078523,
+      "kl": 0.13232421875,
+      "learning_rate": 7.498666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 1876
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.390625,
+      "epoch": 3.0032,
+      "grad_norm": 0.01036189221907158,
+      "kl": 0.14501953125,
+      "learning_rate": 7.497333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1877
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.171875,
+      "epoch": 3.0048,
+      "grad_norm": 0.012276892481833716,
+      "kl": 0.150390625,
+      "learning_rate": 7.496e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1878
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.25,
+      "epoch": 3.0064,
+      "grad_norm": 0.013995210690808113,
+      "kl": 0.13720703125,
+      "learning_rate": 7.494666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1879
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.078125,
+      "epoch": 3.008,
+      "grad_norm": 0.012257433035663077,
+      "kl": 0.134765625,
+      "learning_rate": 7.493333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1880
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.671875,
+      "epoch": 3.0096,
+      "grad_norm": 19.559099683934864,
+      "kl": 0.1435546875,
+      "learning_rate": 7.492e-07,
+      "loss": 0.0001,
+      "reward": 1.84375,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.84375,
+      "step": 1881
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.515625,
+      "epoch": 3.0112,
+      "grad_norm": 0.007102689264530842,
+      "kl": 0.132568359375,
+      "learning_rate": 7.490666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1882
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 105.15625,
+      "epoch": 3.0128,
+      "grad_norm": 0.035493033404483045,
+      "kl": 0.12646484375,
+      "learning_rate": 7.489333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.25,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.25,
+      "step": 1883
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.5625,
+      "epoch": 3.0144,
+      "grad_norm": 0.007650305963870966,
+      "kl": 0.14111328125,
+      "learning_rate": 7.488e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1884
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.8125,
+      "epoch": 3.016,
+      "grad_norm": 0.00862038597240541,
+      "kl": 0.140625,
+      "learning_rate": 7.486666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1885
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.3125,
+      "epoch": 3.0176,
+      "grad_norm": 0.00837846493204736,
+      "kl": 0.133056640625,
+      "learning_rate": 7.485333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1886
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.078125,
+      "epoch": 3.0192,
+      "grad_norm": 0.008970557795382288,
+      "kl": 0.14013671875,
+      "learning_rate": 7.483999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1887
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.421875,
+      "epoch": 3.0208,
+      "grad_norm": 0.007050144988009915,
+      "kl": 0.1328125,
+      "learning_rate": 7.482666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1888
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.859375,
+      "epoch": 3.0224,
+      "grad_norm": 0.016898527438244763,
+      "kl": 0.15478515625,
+      "learning_rate": 7.481333333333333e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1889
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.6875,
+      "epoch": 3.024,
+      "grad_norm": 0.012643855106940065,
+      "kl": 0.1474609375,
+      "learning_rate": 7.48e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1890
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.265625,
+      "epoch": 3.0256,
+      "grad_norm": 1.1472494140344698,
+      "kl": 0.14013671875,
+      "learning_rate": 7.478666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.609375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.625,
+      "step": 1891
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.796875,
+      "epoch": 3.0272,
+      "grad_norm": 0.0086788972943649,
+      "kl": 0.1455078125,
+      "learning_rate": 7.477333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1892
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.375,
+      "epoch": 3.0288,
+      "grad_norm": 0.008778642214261315,
+      "kl": 0.14404296875,
+      "learning_rate": 7.476e-07,
+      "loss": 0.0001,
+      "reward": 1.25,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.25,
+      "step": 1893
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.484375,
+      "epoch": 3.0304,
+      "grad_norm": 0.015544899377417726,
+      "kl": 0.14404296875,
+      "learning_rate": 7.474666666666665e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1894
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.03125,
+      "epoch": 3.032,
+      "grad_norm": 0.0073972009948754785,
+      "kl": 0.13037109375,
+      "learning_rate": 7.473333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1895
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 106.640625,
+      "epoch": 3.0336,
+      "grad_norm": 0.008255350254844722,
+      "kl": 0.147216796875,
+      "learning_rate": 7.471999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1896
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.546875,
+      "epoch": 3.0352,
+      "grad_norm": 0.008667813265873313,
+      "kl": 0.130859375,
+      "learning_rate": 7.470666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1897
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.8125,
+      "epoch": 3.0368,
+      "grad_norm": 0.010375056293413532,
+      "kl": 0.1416015625,
+      "learning_rate": 7.469333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1898
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.328125,
+      "epoch": 3.0384,
+      "grad_norm": 0.007334080737128369,
+      "kl": 0.13330078125,
+      "learning_rate": 7.468e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1899
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.546875,
+      "epoch": 3.04,
+      "grad_norm": 0.006565103786980646,
+      "kl": 0.14013671875,
+      "learning_rate": 7.466666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1900
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.171875,
+      "epoch": 3.0416,
+      "grad_norm": 11.660712492365525,
+      "kl": 0.13916015625,
+      "learning_rate": 7.465333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.640625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.640625,
+      "step": 1901
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.828125,
+      "epoch": 3.0432,
+      "grad_norm": 0.03280884942175404,
+      "kl": 0.1474609375,
+      "learning_rate": 7.464e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1902
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.046875,
+      "epoch": 3.0448,
+      "grad_norm": 5.503506841245263,
+      "kl": 0.13427734375,
+      "learning_rate": 7.462666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.640625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.640625,
+      "step": 1903
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.53125,
+      "epoch": 3.0464,
+      "grad_norm": 0.008795107739896372,
+      "kl": 0.1533203125,
+      "learning_rate": 7.461333333333332e-07,
+      "loss": 0.0002,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1904
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.625,
+      "epoch": 3.048,
+      "grad_norm": 0.012061539347658695,
+      "kl": 0.14111328125,
+      "learning_rate": 7.459999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1905
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 104.15625,
+      "epoch": 3.0496,
+      "grad_norm": 0.007070869562863169,
+      "kl": 0.1552734375,
+      "learning_rate": 7.458666666666666e-07,
+      "loss": 0.0002,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1906
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.140625,
+      "epoch": 3.0512,
+      "grad_norm": 0.006427692842997772,
+      "kl": 0.130126953125,
+      "learning_rate": 7.457333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1907
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.40625,
+      "epoch": 3.0528,
+      "grad_norm": 0.04598896425410621,
+      "kl": 0.1337890625,
+      "learning_rate": 7.456e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1908
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.03125,
+      "epoch": 3.0544,
+      "grad_norm": 0.00838326584858013,
+      "kl": 0.13427734375,
+      "learning_rate": 7.454666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1909
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.890625,
+      "epoch": 3.056,
+      "grad_norm": 0.034162072885561855,
+      "kl": 0.14990234375,
+      "learning_rate": 7.453333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.25,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.25,
+      "step": 1910
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.890625,
+      "epoch": 3.0576,
+      "grad_norm": 0.006044406223033839,
+      "kl": 0.12451171875,
+      "learning_rate": 7.452e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1911
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.640625,
+      "epoch": 3.0592,
+      "grad_norm": 0.008495707751119252,
+      "kl": 0.138671875,
+      "learning_rate": 7.450666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1912
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.515625,
+      "epoch": 3.0608,
+      "grad_norm": 0.006906736710492455,
+      "kl": 0.13525390625,
+      "learning_rate": 7.449333333333333e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 1913
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.890625,
+      "epoch": 3.0624,
+      "grad_norm": 0.06957442348810326,
+      "kl": 0.146484375,
+      "learning_rate": 7.447999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1914
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.9375,
+      "epoch": 3.064,
+      "grad_norm": 0.00695753715183477,
+      "kl": 0.14453125,
+      "learning_rate": 7.446666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1915
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.625,
+      "epoch": 3.0656,
+      "grad_norm": 0.018368772605175037,
+      "kl": 0.13525390625,
+      "learning_rate": 7.445333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1916
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.28125,
+      "epoch": 3.0672,
+      "grad_norm": 0.00927695883912688,
+      "kl": 0.14111328125,
+      "learning_rate": 7.443999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1917
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.625,
+      "epoch": 3.0688,
+      "grad_norm": 0.0087966138551967,
+      "kl": 0.117919921875,
+      "learning_rate": 7.442666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1918
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.390625,
+      "epoch": 3.0704,
+      "grad_norm": 0.020909459513362393,
+      "kl": 0.13916015625,
+      "learning_rate": 7.441333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1919
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.828125,
+      "epoch": 3.072,
+      "grad_norm": 0.007558676707739958,
+      "kl": 0.134765625,
+      "learning_rate": 7.44e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1920
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.046875,
+      "epoch": 3.0736,
+      "grad_norm": 0.021690479422078215,
+      "kl": 0.1328125,
+      "learning_rate": 7.438666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1921
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 107.234375,
+      "epoch": 3.0752,
+      "grad_norm": 9.515925311219526,
+      "kl": 0.13134765625,
+      "learning_rate": 7.437333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.765625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.765625,
+      "step": 1922
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.953125,
+      "epoch": 3.0768,
+      "grad_norm": 0.014698083904335909,
+      "kl": 0.1181640625,
+      "learning_rate": 7.436e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1923
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.5625,
+      "epoch": 3.0784,
+      "grad_norm": 22.012691613202563,
+      "kl": 0.12646484375,
+      "learning_rate": 7.434666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.28125,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.28125,
+      "step": 1924
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.5,
+      "epoch": 3.08,
+      "grad_norm": 0.00939744257132282,
+      "kl": 0.13330078125,
+      "learning_rate": 7.433333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1925
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.03125,
+      "epoch": 3.0816,
+      "grad_norm": 0.007714162182279978,
+      "kl": 0.122314453125,
+      "learning_rate": 7.431999999999999e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 1926
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.625,
+      "epoch": 3.0832,
+      "grad_norm": 0.007681580636417243,
+      "kl": 0.13623046875,
+      "learning_rate": 7.430666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1927
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.359375,
+      "epoch": 3.0848,
+      "grad_norm": 0.015159945881359292,
+      "kl": 0.118896484375,
+      "learning_rate": 7.429333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1928
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.3125,
+      "epoch": 3.0864,
+      "grad_norm": 2.3222831155234243,
+      "kl": 0.113525390625,
+      "learning_rate": 7.428e-07,
+      "loss": 0.0001,
+      "reward": 1.609375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.625,
+      "step": 1929
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.765625,
+      "epoch": 3.088,
+      "grad_norm": 0.008988948737374934,
+      "kl": 0.1181640625,
+      "learning_rate": 7.426666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1930
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.140625,
+      "epoch": 3.0896,
+      "grad_norm": 0.008264366957981117,
+      "kl": 0.12744140625,
+      "learning_rate": 7.425333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1931
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.390625,
+      "epoch": 3.0912,
+      "grad_norm": 0.005601800157839357,
+      "kl": 0.105224609375,
+      "learning_rate": 7.423999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1932
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.40625,
+      "epoch": 3.0928,
+      "grad_norm": 0.007969445465066151,
+      "kl": 0.1123046875,
+      "learning_rate": 7.422666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1933
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.78125,
+      "epoch": 3.0944,
+      "grad_norm": 0.0065481569559090935,
+      "kl": 0.114013671875,
+      "learning_rate": 7.421333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1934
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.21875,
+      "epoch": 3.096,
+      "grad_norm": 0.008110407098033613,
+      "kl": 0.111083984375,
+      "learning_rate": 7.42e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1935
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.921875,
+      "epoch": 3.0976,
+      "grad_norm": 0.007099587672400425,
+      "kl": 0.11083984375,
+      "learning_rate": 7.418666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1936
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.671875,
+      "epoch": 3.0992,
+      "grad_norm": 0.006167773804670096,
+      "kl": 0.1123046875,
+      "learning_rate": 7.417333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1937
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.578125,
+      "epoch": 3.1008,
+      "grad_norm": 0.007916934413788695,
+      "kl": 0.114501953125,
+      "learning_rate": 7.416e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1938
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.140625,
+      "epoch": 3.1024,
+      "grad_norm": 0.8471622405590179,
+      "kl": 0.117919921875,
+      "learning_rate": 7.414666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.59375,
+      "reward_std": 0.0883883461356163,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.609375,
+      "step": 1939
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.171875,
+      "epoch": 3.104,
+      "grad_norm": 0.005474708804567338,
+      "kl": 0.101318359375,
+      "learning_rate": 7.413333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1940
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.03125,
+      "epoch": 3.1056,
+      "grad_norm": 0.013188078711307921,
+      "kl": 0.12255859375,
+      "learning_rate": 7.411999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1941
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.578125,
+      "epoch": 3.1072,
+      "grad_norm": 0.009920855311360234,
+      "kl": 0.119384765625,
+      "learning_rate": 7.410666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1942
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.53125,
+      "epoch": 3.1088,
+      "grad_norm": 0.005833032478170943,
+      "kl": 0.116455078125,
+      "learning_rate": 7.409333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1943
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.53125,
+      "epoch": 3.1104,
+      "grad_norm": 0.00541887196137913,
+      "kl": 0.114013671875,
+      "learning_rate": 7.408e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 1944
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.453125,
+      "epoch": 3.112,
+      "grad_norm": 0.008126770120312574,
+      "kl": 0.112548828125,
+      "learning_rate": 7.406666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1945
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.53125,
+      "epoch": 3.1136,
+      "grad_norm": 0.03220354652434343,
+      "kl": 0.14111328125,
+      "learning_rate": 7.405333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1946
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.8125,
+      "epoch": 3.1152,
+      "grad_norm": 8.672110634724298,
+      "kl": 0.119140625,
+      "learning_rate": 7.403999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.984375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.984375,
+      "step": 1947
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.984375,
+      "epoch": 3.1168,
+      "grad_norm": 0.009405295066479897,
+      "kl": 0.126708984375,
+      "learning_rate": 7.402666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1948
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.953125,
+      "epoch": 3.1184,
+      "grad_norm": 0.005640857266634208,
+      "kl": 0.118408203125,
+      "learning_rate": 7.401333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1949
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.859375,
+      "epoch": 3.12,
+      "grad_norm": 0.005402937873154549,
+      "kl": 0.118408203125,
+      "learning_rate": 7.4e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1950
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.640625,
+      "epoch": 3.1216,
+      "grad_norm": 0.006577436759757455,
+      "kl": 0.119140625,
+      "learning_rate": 7.398666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1951
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.28125,
+      "epoch": 3.1232,
+      "grad_norm": 0.008702020832940166,
+      "kl": 0.130859375,
+      "learning_rate": 7.397333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1952
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.859375,
+      "epoch": 3.1248,
+      "grad_norm": 0.008021219812609642,
+      "kl": 0.12255859375,
+      "learning_rate": 7.396e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1953
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.75,
+      "epoch": 3.1264,
+      "grad_norm": 0.01130670493008822,
+      "kl": 0.124267578125,
+      "learning_rate": 7.394666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1954
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.84375,
+      "epoch": 3.128,
+      "grad_norm": 0.011268268723796103,
+      "kl": 0.130126953125,
+      "learning_rate": 7.393333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1955
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.921875,
+      "epoch": 3.1296,
+      "grad_norm": 0.005906471779438174,
+      "kl": 0.1220703125,
+      "learning_rate": 7.392e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1956
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.03125,
+      "epoch": 3.1312,
+      "grad_norm": 0.009649612696367514,
+      "kl": 0.1240234375,
+      "learning_rate": 7.390666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1957
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.78125,
+      "epoch": 3.1328,
+      "grad_norm": 0.02365125583754476,
+      "kl": 0.1318359375,
+      "learning_rate": 7.389333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1958
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.171875,
+      "epoch": 3.1344,
+      "grad_norm": 0.0060794148036322476,
+      "kl": 0.115966796875,
+      "learning_rate": 7.388e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1959
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.71875,
+      "epoch": 3.136,
+      "grad_norm": 0.023705493099049818,
+      "kl": 0.126953125,
+      "learning_rate": 7.386666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1960
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.515625,
+      "epoch": 3.1376,
+      "grad_norm": 0.00805618755252714,
+      "kl": 0.12158203125,
+      "learning_rate": 7.385333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1961
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.65625,
+      "epoch": 3.1391999999999998,
+      "grad_norm": 0.007047150684785592,
+      "kl": 0.1201171875,
+      "learning_rate": 7.383999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1962
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.921875,
+      "epoch": 3.1408,
+      "grad_norm": 0.008506294732715481,
+      "kl": 0.129150390625,
+      "learning_rate": 7.382666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1963
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.859375,
+      "epoch": 3.1424,
+      "grad_norm": 0.006891208777521889,
+      "kl": 0.12353515625,
+      "learning_rate": 7.381333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1964
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.53125,
+      "epoch": 3.144,
+      "grad_norm": 0.007165678003220649,
+      "kl": 0.128173828125,
+      "learning_rate": 7.38e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1965
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.984375,
+      "epoch": 3.1456,
+      "grad_norm": 0.0059681894154949115,
+      "kl": 0.119384765625,
+      "learning_rate": 7.378666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1966
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.296875,
+      "epoch": 3.1471999999999998,
+      "grad_norm": 0.02038280222609716,
+      "kl": 0.117919921875,
+      "learning_rate": 7.377333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1967
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.078125,
+      "epoch": 3.1488,
+      "grad_norm": 4.004678650235036,
+      "kl": 0.130615234375,
+      "learning_rate": 7.376e-07,
+      "loss": 0.0001,
+      "reward": 1.640625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.640625,
+      "step": 1968
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.328125,
+      "epoch": 3.1504,
+      "grad_norm": 0.008432504742220081,
+      "kl": 0.119873046875,
+      "learning_rate": 7.374666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1969
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.8125,
+      "epoch": 3.152,
+      "grad_norm": 0.007725747180095309,
+      "kl": 0.12353515625,
+      "learning_rate": 7.373333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1970
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.03125,
+      "epoch": 3.1536,
+      "grad_norm": 15.98873542149025,
+      "kl": 0.140625,
+      "learning_rate": 7.371999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.53125,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.53125,
+      "step": 1971
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.875,
+      "epoch": 3.1552,
+      "grad_norm": 16.177521876932644,
+      "kl": 0.130859375,
+      "learning_rate": 7.370666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.515625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.515625,
+      "step": 1972
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.703125,
+      "epoch": 3.1568,
+      "grad_norm": 3.4858720272617463,
+      "kl": 0.1171875,
+      "learning_rate": 7.369333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.46875,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.46875,
+      "step": 1973
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.9375,
+      "epoch": 3.1584,
+      "grad_norm": 0.007022540111422096,
+      "kl": 0.109130859375,
+      "learning_rate": 7.368e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 1974
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.046875,
+      "epoch": 3.16,
+      "grad_norm": 0.01934759301898413,
+      "kl": 0.1220703125,
+      "learning_rate": 7.366666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1975
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.125,
+      "epoch": 3.1616,
+      "grad_norm": 0.016235360418796037,
+      "kl": 0.117431640625,
+      "learning_rate": 7.365333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1976
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.8125,
+      "epoch": 3.1632,
+      "grad_norm": 0.018754922204061695,
+      "kl": 0.107177734375,
+      "learning_rate": 7.364000000000001e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1977
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.734375,
+      "epoch": 3.1648,
+      "grad_norm": 0.00734389110059404,
+      "kl": 0.10693359375,
+      "learning_rate": 7.362666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1978
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.8125,
+      "epoch": 3.1664,
+      "grad_norm": 0.01721485840973681,
+      "kl": 0.133544921875,
+      "learning_rate": 7.361333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1979
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.75,
+      "epoch": 3.168,
+      "grad_norm": 0.005864426075237538,
+      "kl": 0.112548828125,
+      "learning_rate": 7.359999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1980
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.34375,
+      "epoch": 3.1696,
+      "grad_norm": 0.005814881608158245,
+      "kl": 0.109619140625,
+      "learning_rate": 7.358666666666666e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 1981
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.5,
+      "epoch": 3.1712,
+      "grad_norm": 0.006775416980989863,
+      "kl": 0.107421875,
+      "learning_rate": 7.357333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 1982
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.640625,
+      "epoch": 3.1728,
+      "grad_norm": 19.779625795335953,
+      "kl": 0.11669921875,
+      "learning_rate": 7.356e-07,
+      "loss": 0.0001,
+      "reward": 1.53125,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.53125,
+      "step": 1983
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.59375,
+      "epoch": 3.1744,
+      "grad_norm": 0.014901010493549945,
+      "kl": 0.115478515625,
+      "learning_rate": 7.354666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1984
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.71875,
+      "epoch": 3.176,
+      "grad_norm": 0.0063971064003203095,
+      "kl": 0.109375,
+      "learning_rate": 7.353333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 1985
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.75,
+      "epoch": 3.1776,
+      "grad_norm": 9.609061810714817,
+      "kl": 0.11083984375,
+      "learning_rate": 7.352e-07,
+      "loss": 0.0001,
+      "reward": 1.53125,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.53125,
+      "step": 1986
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.5,
+      "epoch": 3.1792,
+      "grad_norm": 0.008443473161842139,
+      "kl": 0.10888671875,
+      "learning_rate": 7.350666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1987
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.078125,
+      "epoch": 3.1808,
+      "grad_norm": 0.0060259221573050045,
+      "kl": 0.108642578125,
+      "learning_rate": 7.349333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 1988
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.625,
+      "epoch": 3.1824,
+      "grad_norm": 0.010213532236474948,
+      "kl": 0.11474609375,
+      "learning_rate": 7.347999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1989
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.03125,
+      "epoch": 3.184,
+      "grad_norm": 0.006233981272471328,
+      "kl": 0.111328125,
+      "learning_rate": 7.346666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1990
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.421875,
+      "epoch": 3.1856,
+      "grad_norm": 0.008718857593088231,
+      "kl": 0.10546875,
+      "learning_rate": 7.345333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1991
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.75,
+      "epoch": 3.1872,
+      "grad_norm": 0.007109996452873717,
+      "kl": 0.103759765625,
+      "learning_rate": 7.344e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1992
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.78125,
+      "epoch": 3.1888,
+      "grad_norm": 0.00496526063561423,
+      "kl": 0.099853515625,
+      "learning_rate": 7.342666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1993
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.34375,
+      "epoch": 3.1904,
+      "grad_norm": 0.006949014602593719,
+      "kl": 0.116943359375,
+      "learning_rate": 7.341333333333333e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 1994
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.859375,
+      "epoch": 3.192,
+      "grad_norm": 0.01985223688113867,
+      "kl": 0.10595703125,
+      "learning_rate": 7.34e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1995
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.109375,
+      "epoch": 3.1936,
+      "grad_norm": 11.906912745878333,
+      "kl": 0.1162109375,
+      "learning_rate": 7.338666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 1996
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.1875,
+      "epoch": 3.1952,
+      "grad_norm": 0.006507738211405048,
+      "kl": 0.113525390625,
+      "learning_rate": 7.337333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1997
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.25,
+      "epoch": 3.1968,
+      "grad_norm": 0.0053615155285117195,
+      "kl": 0.109375,
+      "learning_rate": 7.336e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 1998
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.953125,
+      "epoch": 3.1984,
+      "grad_norm": 0.00791915824168357,
+      "kl": 0.1123046875,
+      "learning_rate": 7.334666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 1999
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.65625,
+      "epoch": 3.2,
+      "grad_norm": 0.007554813991394576,
+      "kl": 0.11328125,
+      "learning_rate": 7.333333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2000
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.015625,
+      "epoch": 3.2016,
+      "grad_norm": 10.625261572125734,
+      "kl": 0.1064453125,
+      "learning_rate": 7.331999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.53125,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.53125,
+      "step": 2001
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.40625,
+      "epoch": 3.2032,
+      "grad_norm": 12.883747506953014,
+      "kl": 0.108154296875,
+      "learning_rate": 7.330666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.8125,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.8125,
+      "step": 2002
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.5625,
+      "epoch": 3.2048,
+      "grad_norm": 0.027208907508713513,
+      "kl": 0.10498046875,
+      "learning_rate": 7.329333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2003
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.375,
+      "epoch": 3.2064,
+      "grad_norm": 0.006328791974617841,
+      "kl": 0.1201171875,
+      "learning_rate": 7.328e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2004
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.78125,
+      "epoch": 3.208,
+      "grad_norm": 0.008615798916059028,
+      "kl": 0.10693359375,
+      "learning_rate": 7.326666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 2005
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.765625,
+      "epoch": 3.2096,
+      "grad_norm": 0.00610858635386921,
+      "kl": 0.10986328125,
+      "learning_rate": 7.325333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2006
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.875,
+      "epoch": 3.2112,
+      "grad_norm": 0.0067682595646023315,
+      "kl": 0.1259765625,
+      "learning_rate": 7.324e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2007
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.234375,
+      "epoch": 3.2128,
+      "grad_norm": 5.117932592565424,
+      "kl": 0.109375,
+      "learning_rate": 7.322666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.515625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.515625,
+      "step": 2008
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.171875,
+      "epoch": 3.2144,
+      "grad_norm": 0.005871968181778277,
+      "kl": 0.105712890625,
+      "learning_rate": 7.321333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2009
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.609375,
+      "epoch": 3.216,
+      "grad_norm": 0.029715051576496298,
+      "kl": 0.1123046875,
+      "learning_rate": 7.319999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2010
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.703125,
+      "epoch": 3.2176,
+      "grad_norm": 8.412518679678147,
+      "kl": 0.115234375,
+      "learning_rate": 7.318666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.671875,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.671875,
+      "step": 2011
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.609375,
+      "epoch": 3.2192,
+      "grad_norm": 0.007576459739268805,
+      "kl": 0.10888671875,
+      "learning_rate": 7.317333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2012
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.25,
+      "epoch": 3.2208,
+      "grad_norm": 10.35385428874573,
+      "kl": 0.1123046875,
+      "learning_rate": 7.316e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.1157275140285492,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2013
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.640625,
+      "epoch": 3.2224,
+      "grad_norm": 0.011095619840819663,
+      "kl": 0.11376953125,
+      "learning_rate": 7.314666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2014
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.75,
+      "epoch": 3.224,
+      "grad_norm": 2.595366883663525,
+      "kl": 0.114990234375,
+      "learning_rate": 7.313333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0883883461356163,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2015
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.25,
+      "epoch": 3.2256,
+      "grad_norm": 0.009159148152043654,
+      "kl": 0.100341796875,
+      "learning_rate": 7.311999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2016
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.609375,
+      "epoch": 3.2272,
+      "grad_norm": 0.006001435916743212,
+      "kl": 0.11962890625,
+      "learning_rate": 7.310666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2017
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.21875,
+      "epoch": 3.2288,
+      "grad_norm": 6.957498105987208,
+      "kl": 0.1044921875,
+      "learning_rate": 7.309333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.765625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.765625,
+      "step": 2018
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 141.765625,
+      "epoch": 3.2304,
+      "grad_norm": 0.01735564031593225,
+      "kl": 0.12158203125,
+      "learning_rate": 7.308e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2019
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.421875,
+      "epoch": 3.232,
+      "grad_norm": 0.00769689304817737,
+      "kl": 0.100830078125,
+      "learning_rate": 7.306666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2020
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.59375,
+      "epoch": 3.2336,
+      "grad_norm": 0.010100686022749652,
+      "kl": 0.107666015625,
+      "learning_rate": 7.305333333333333e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 2021
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.984375,
+      "epoch": 3.2352,
+      "grad_norm": 0.005839762251804912,
+      "kl": 0.113525390625,
+      "learning_rate": 7.304e-07,
+      "loss": 0.0001,
+      "reward": 1.25,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.25,
+      "step": 2022
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 141.5,
+      "epoch": 3.2368,
+      "grad_norm": 0.005789065305967974,
+      "kl": 0.111083984375,
+      "learning_rate": 7.302666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2023
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.4375,
+      "epoch": 3.2384,
+      "grad_norm": 0.005217415289121038,
+      "kl": 0.10302734375,
+      "learning_rate": 7.301333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2024
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 142.578125,
+      "epoch": 3.24,
+      "grad_norm": 0.00529730772124171,
+      "kl": 0.11376953125,
+      "learning_rate": 7.3e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2025
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.546875,
+      "epoch": 3.2416,
+      "grad_norm": 11.418835016310453,
+      "kl": 0.102294921875,
+      "learning_rate": 7.298666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.90625,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.90625,
+      "step": 2026
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.609375,
+      "epoch": 3.2432,
+      "grad_norm": 14.794853524477801,
+      "kl": 0.10205078125,
+      "learning_rate": 7.297333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5625,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5625,
+      "step": 2027
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 142.28125,
+      "epoch": 3.2448,
+      "grad_norm": 0.006699489810302841,
+      "kl": 0.106201171875,
+      "learning_rate": 7.296e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2028
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 142.53125,
+      "epoch": 3.2464,
+      "grad_norm": 0.03902941622853143,
+      "kl": 0.10693359375,
+      "learning_rate": 7.294666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2029
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 143.03125,
+      "epoch": 3.248,
+      "grad_norm": 0.8934875284908791,
+      "kl": 0.115478515625,
+      "learning_rate": 7.293333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.96875,
+      "reward_std": 0.0883883461356163,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.984375,
+      "step": 2030
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.09375,
+      "epoch": 3.2496,
+      "grad_norm": 0.011661965296252333,
+      "kl": 0.114990234375,
+      "learning_rate": 7.291999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2031
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.59375,
+      "epoch": 3.2512,
+      "grad_norm": 0.00955910021775449,
+      "kl": 0.10595703125,
+      "learning_rate": 7.290666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2032
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.046875,
+      "epoch": 3.2528,
+      "grad_norm": 0.008141236338556663,
+      "kl": 0.106689453125,
+      "learning_rate": 7.289333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2033
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.8125,
+      "epoch": 3.2544,
+      "grad_norm": 0.14674669901747917,
+      "kl": 0.10888671875,
+      "learning_rate": 7.288e-07,
+      "loss": 0.0001,
+      "reward": 1.25,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.25,
+      "step": 2034
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.84375,
+      "epoch": 3.2560000000000002,
+      "grad_norm": 0.006707597545206179,
+      "kl": 0.114990234375,
+      "learning_rate": 7.286666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2035
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.328125,
+      "epoch": 3.2576,
+      "grad_norm": 0.007413559466707846,
+      "kl": 0.1083984375,
+      "learning_rate": 7.285333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2036
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.421875,
+      "epoch": 3.2592,
+      "grad_norm": 4.533786358138749,
+      "kl": 0.1103515625,
+      "learning_rate": 7.284e-07,
+      "loss": 0.0001,
+      "reward": 1.609375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.609375,
+      "step": 2037
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.1875,
+      "epoch": 3.2608,
+      "grad_norm": 0.011117318392609791,
+      "kl": 0.110595703125,
+      "learning_rate": 7.282666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2038
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.65625,
+      "epoch": 3.2624,
+      "grad_norm": 0.008712360824108867,
+      "kl": 0.108154296875,
+      "learning_rate": 7.281333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2039
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.84375,
+      "epoch": 3.2640000000000002,
+      "grad_norm": 0.005921627151499945,
+      "kl": 0.13330078125,
+      "learning_rate": 7.28e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2040
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.96875,
+      "epoch": 3.2656,
+      "grad_norm": 0.014333486059744315,
+      "kl": 0.116455078125,
+      "learning_rate": 7.278666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 2041
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.84375,
+      "epoch": 3.2672,
+      "grad_norm": 0.005258228629888007,
+      "kl": 0.11181640625,
+      "learning_rate": 7.277333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2042
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.296875,
+      "epoch": 3.2688,
+      "grad_norm": 0.007390894062658885,
+      "kl": 0.118408203125,
+      "learning_rate": 7.276e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2043
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.640625,
+      "epoch": 3.2704,
+      "grad_norm": 0.006576887301871694,
+      "kl": 0.13037109375,
+      "learning_rate": 7.274666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2044
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.875,
+      "epoch": 3.2720000000000002,
+      "grad_norm": 0.0058233123503460315,
+      "kl": 0.1220703125,
+      "learning_rate": 7.273333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2045
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.609375,
+      "epoch": 3.2736,
+      "grad_norm": 0.009715181595815098,
+      "kl": 0.120849609375,
+      "learning_rate": 7.271999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2046
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.390625,
+      "epoch": 3.2752,
+      "grad_norm": 0.006317255035016137,
+      "kl": 0.12255859375,
+      "learning_rate": 7.270666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2047
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.328125,
+      "epoch": 3.2768,
+      "grad_norm": 0.006052032413449983,
+      "kl": 0.11328125,
+      "learning_rate": 7.269333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2048
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.890625,
+      "epoch": 3.2784,
+      "grad_norm": 0.0060478715035959885,
+      "kl": 0.114013671875,
+      "learning_rate": 7.268e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2049
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 112.78125,
+      "epoch": 3.2800000000000002,
+      "grad_norm": 0.007273326570669428,
+      "kl": 0.125244140625,
+      "learning_rate": 7.266666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.25,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.25,
+      "step": 2050
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.96875,
+      "epoch": 3.2816,
+      "grad_norm": 0.00542831838385398,
+      "kl": 0.122802734375,
+      "learning_rate": 7.265333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2051
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.15625,
+      "epoch": 3.2832,
+      "grad_norm": 0.00812253656439316,
+      "kl": 0.112060546875,
+      "learning_rate": 7.264e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2052
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.265625,
+      "epoch": 3.2848,
+      "grad_norm": 0.007461985510716034,
+      "kl": 0.1279296875,
+      "learning_rate": 7.262666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2053
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.5625,
+      "epoch": 3.2864,
+      "grad_norm": 0.00626028261705363,
+      "kl": 0.123046875,
+      "learning_rate": 7.261333333333332e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 2054
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.59375,
+      "epoch": 3.288,
+      "grad_norm": 0.008058481591860853,
+      "kl": 0.11279296875,
+      "learning_rate": 7.259999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2055
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.140625,
+      "epoch": 3.2896,
+      "grad_norm": 0.07887627694646751,
+      "kl": 0.10986328125,
+      "learning_rate": 7.258666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2056
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.171875,
+      "epoch": 3.2912,
+      "grad_norm": 0.010476335858109887,
+      "kl": 0.11669921875,
+      "learning_rate": 7.257333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2057
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.71875,
+      "epoch": 3.2928,
+      "grad_norm": 5.722710828343012,
+      "kl": 0.1220703125,
+      "learning_rate": 7.256e-07,
+      "loss": 0.0001,
+      "reward": 1.828125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.828125,
+      "step": 2058
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.390625,
+      "epoch": 3.2944,
+      "grad_norm": 0.005237846444068062,
+      "kl": 0.11376953125,
+      "learning_rate": 7.254666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2059
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.1875,
+      "epoch": 3.296,
+      "grad_norm": 0.006970314115902089,
+      "kl": 0.109375,
+      "learning_rate": 7.253333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2060
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.3125,
+      "epoch": 3.2976,
+      "grad_norm": 0.007853891354922206,
+      "kl": 0.120361328125,
+      "learning_rate": 7.252e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2061
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.859375,
+      "epoch": 3.2992,
+      "grad_norm": 0.005768779531302082,
+      "kl": 0.110595703125,
+      "learning_rate": 7.250666666666666e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 2062
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.65625,
+      "epoch": 3.3008,
+      "grad_norm": 0.006533172810821031,
+      "kl": 0.116455078125,
+      "learning_rate": 7.249333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2063
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.859375,
+      "epoch": 3.3024,
+      "grad_norm": 0.0069394818744340225,
+      "kl": 0.11474609375,
+      "learning_rate": 7.247999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2064
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.296875,
+      "epoch": 3.304,
+      "grad_norm": 0.00761718582646049,
+      "kl": 0.11865234375,
+      "learning_rate": 7.246666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2065
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.984375,
+      "epoch": 3.3056,
+      "grad_norm": 0.0059853941283284215,
+      "kl": 0.115234375,
+      "learning_rate": 7.245333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2066
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.71875,
+      "epoch": 3.3072,
+      "grad_norm": 0.006413173247632955,
+      "kl": 0.1328125,
+      "learning_rate": 7.244e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2067
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 108.765625,
+      "epoch": 3.3088,
+      "grad_norm": 0.00763218298064406,
+      "kl": 0.11767578125,
+      "learning_rate": 7.242666666666666e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 2068
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.0,
+      "epoch": 3.3104,
+      "grad_norm": 0.005386983072661571,
+      "kl": 0.11474609375,
+      "learning_rate": 7.241333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2069
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.671875,
+      "epoch": 3.312,
+      "grad_norm": 0.0060979247998294035,
+      "kl": 0.125,
+      "learning_rate": 7.24e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2070
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.75,
+      "epoch": 3.3136,
+      "grad_norm": 0.013235301541440932,
+      "kl": 0.125732421875,
+      "learning_rate": 7.238666666666667e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 2071
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.4375,
+      "epoch": 3.3152,
+      "grad_norm": 0.008066087287407727,
+      "kl": 0.13232421875,
+      "learning_rate": 7.237333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2072
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.421875,
+      "epoch": 3.3168,
+      "grad_norm": 0.006238746469111423,
+      "kl": 0.113525390625,
+      "learning_rate": 7.235999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2073
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.40625,
+      "epoch": 3.3184,
+      "grad_norm": 35.93408159506594,
+      "kl": 0.116455078125,
+      "learning_rate": 7.234666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.46875,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.46875,
+      "step": 2074
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 110.078125,
+      "epoch": 3.32,
+      "grad_norm": 0.005258066149136759,
+      "kl": 0.115234375,
+      "learning_rate": 7.233333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2075
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.21875,
+      "epoch": 3.3216,
+      "grad_norm": 0.026330413360627587,
+      "kl": 0.11328125,
+      "learning_rate": 7.231999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2076
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.484375,
+      "epoch": 3.3232,
+      "grad_norm": 0.02116941013436171,
+      "kl": 0.1142578125,
+      "learning_rate": 7.230666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2077
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.078125,
+      "epoch": 3.3247999999999998,
+      "grad_norm": 0.006926347715438975,
+      "kl": 0.118896484375,
+      "learning_rate": 7.229333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2078
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.203125,
+      "epoch": 3.3264,
+      "grad_norm": 0.0053050434819471,
+      "kl": 0.100341796875,
+      "learning_rate": 7.228e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2079
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.921875,
+      "epoch": 3.328,
+      "grad_norm": 0.034505376790987786,
+      "kl": 0.10693359375,
+      "learning_rate": 7.226666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2080
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.78125,
+      "epoch": 3.3296,
+      "grad_norm": 6.860250176808265,
+      "kl": 0.11376953125,
+      "learning_rate": 7.225333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.984375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.984375,
+      "step": 2081
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.65625,
+      "epoch": 3.3312,
+      "grad_norm": 0.03851142710222732,
+      "kl": 0.11181640625,
+      "learning_rate": 7.224e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2082
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.453125,
+      "epoch": 3.3327999999999998,
+      "grad_norm": 0.0060336912110086375,
+      "kl": 0.111328125,
+      "learning_rate": 7.222666666666665e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2083
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.65625,
+      "epoch": 3.3344,
+      "grad_norm": 0.02001856641129675,
+      "kl": 0.117919921875,
+      "learning_rate": 7.221333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2084
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.53125,
+      "epoch": 3.336,
+      "grad_norm": 0.006775606145633515,
+      "kl": 0.12353515625,
+      "learning_rate": 7.219999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2085
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.390625,
+      "epoch": 3.3376,
+      "grad_norm": 0.005655517390752979,
+      "kl": 0.116455078125,
+      "learning_rate": 7.218666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2086
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.703125,
+      "epoch": 3.3392,
+      "grad_norm": 0.022268582759844273,
+      "kl": 0.11376953125,
+      "learning_rate": 7.217333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2087
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.4375,
+      "epoch": 3.3407999999999998,
+      "grad_norm": 0.005385767144514793,
+      "kl": 0.10546875,
+      "learning_rate": 7.216e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2088
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 109.734375,
+      "epoch": 3.3424,
+      "grad_norm": 0.01026744223129377,
+      "kl": 0.11328125,
+      "learning_rate": 7.214666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2089
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.890625,
+      "epoch": 3.344,
+      "grad_norm": 0.00723835701633237,
+      "kl": 0.114501953125,
+      "learning_rate": 7.213333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2090
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.484375,
+      "epoch": 3.3456,
+      "grad_norm": 0.018699317838864665,
+      "kl": 0.125732421875,
+      "learning_rate": 7.211999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2091
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.046875,
+      "epoch": 3.3472,
+      "grad_norm": 0.0049089111629642525,
+      "kl": 0.1025390625,
+      "learning_rate": 7.210666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2092
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.03125,
+      "epoch": 3.3487999999999998,
+      "grad_norm": 0.0065780322701599345,
+      "kl": 0.10791015625,
+      "learning_rate": 7.209333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.25,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.25,
+      "step": 2093
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.53125,
+      "epoch": 3.3504,
+      "grad_norm": 0.005668207242419259,
+      "kl": 0.104248046875,
+      "learning_rate": 7.207999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2094
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.25,
+      "epoch": 3.352,
+      "grad_norm": 0.005404623971505458,
+      "kl": 0.111328125,
+      "learning_rate": 7.206666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2095
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 113.515625,
+      "epoch": 3.3536,
+      "grad_norm": 0.0060993428899606,
+      "kl": 0.115234375,
+      "learning_rate": 7.205333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2096
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.4375,
+      "epoch": 3.3552,
+      "grad_norm": 0.005177191193853862,
+      "kl": 0.10888671875,
+      "learning_rate": 7.204e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 2097
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.390625,
+      "epoch": 3.3568,
+      "grad_norm": 0.005273029295266944,
+      "kl": 0.109130859375,
+      "learning_rate": 7.202666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2098
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.421875,
+      "epoch": 3.3584,
+      "grad_norm": 0.008368511357299609,
+      "kl": 0.102294921875,
+      "learning_rate": 7.201333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2099
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.703125,
+      "epoch": 3.36,
+      "grad_norm": 0.0075110861870411735,
+      "kl": 0.111083984375,
+      "learning_rate": 7.2e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2100
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.421875,
+      "epoch": 3.3616,
+      "grad_norm": 0.005074059513133587,
+      "kl": 0.115234375,
+      "learning_rate": 7.198666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2101
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.921875,
+      "epoch": 3.3632,
+      "grad_norm": 0.006395548690828237,
+      "kl": 0.10595703125,
+      "learning_rate": 7.197333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2102
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.15625,
+      "epoch": 3.3648,
+      "grad_norm": 0.005277441892789133,
+      "kl": 0.105224609375,
+      "learning_rate": 7.196e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2103
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.15625,
+      "epoch": 3.3664,
+      "grad_norm": 0.0045519793848087554,
+      "kl": 0.1005859375,
+      "learning_rate": 7.194666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2104
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.828125,
+      "epoch": 3.368,
+      "grad_norm": 0.010428395427995715,
+      "kl": 0.106689453125,
+      "learning_rate": 7.193333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2105
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.03125,
+      "epoch": 3.3696,
+      "grad_norm": 0.005913686389410635,
+      "kl": 0.1025390625,
+      "learning_rate": 7.191999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2106
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.640625,
+      "epoch": 3.3712,
+      "grad_norm": 0.006811668936428607,
+      "kl": 0.109130859375,
+      "learning_rate": 7.190666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2107
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.28125,
+      "epoch": 3.3728,
+      "grad_norm": 73.58244234016462,
+      "kl": 0.107177734375,
+      "learning_rate": 7.189333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.640625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.640625,
+      "step": 2108
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.15625,
+      "epoch": 3.3744,
+      "grad_norm": 0.008349293079073774,
+      "kl": 0.1064453125,
+      "learning_rate": 7.188e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2109
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.15625,
+      "epoch": 3.376,
+      "grad_norm": 0.005768462290269828,
+      "kl": 0.1103515625,
+      "learning_rate": 7.186666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2110
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.484375,
+      "epoch": 3.3776,
+      "grad_norm": 0.006856799479007221,
+      "kl": 0.10498046875,
+      "learning_rate": 7.185333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2111
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.75,
+      "epoch": 3.3792,
+      "grad_norm": 0.005352857275126046,
+      "kl": 0.1083984375,
+      "learning_rate": 7.184e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 2112
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.5625,
+      "epoch": 3.3808,
+      "grad_norm": 0.004911680952407318,
+      "kl": 0.103759765625,
+      "learning_rate": 7.182666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2113
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.484375,
+      "epoch": 3.3824,
+      "grad_norm": 0.0050329802002596355,
+      "kl": 0.097412109375,
+      "learning_rate": 7.181333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2114
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.421875,
+      "epoch": 3.384,
+      "grad_norm": 0.006998273762814341,
+      "kl": 0.1025390625,
+      "learning_rate": 7.179999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2115
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.953125,
+      "epoch": 3.3856,
+      "grad_norm": 0.011349973687311565,
+      "kl": 0.107666015625,
+      "learning_rate": 7.178666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2116
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.953125,
+      "epoch": 3.3872,
+      "grad_norm": 7.3622095262332055,
+      "kl": 0.100341796875,
+      "learning_rate": 7.177333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.78125,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.78125,
+      "step": 2117
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.609375,
+      "epoch": 3.3888,
+      "grad_norm": 75.42063454245998,
+      "kl": 0.108642578125,
+      "learning_rate": 7.176e-07,
+      "loss": 0.0001,
+      "reward": 1.59375,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.59375,
+      "step": 2118
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.921875,
+      "epoch": 3.3904,
+      "grad_norm": 0.004903221936812277,
+      "kl": 0.11083984375,
+      "learning_rate": 7.174666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2119
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.84375,
+      "epoch": 3.392,
+      "grad_norm": 0.004573320500508863,
+      "kl": 0.0966796875,
+      "learning_rate": 7.173333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2120
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.75,
+      "epoch": 3.3936,
+      "grad_norm": 0.00674017653844564,
+      "kl": 0.10693359375,
+      "learning_rate": 7.171999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2121
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.921875,
+      "epoch": 3.3952,
+      "grad_norm": 0.00504798160447893,
+      "kl": 0.099365234375,
+      "learning_rate": 7.170666666666666e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 2122
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.5625,
+      "epoch": 3.3968,
+      "grad_norm": 0.006180617738121244,
+      "kl": 0.10009765625,
+      "learning_rate": 7.169333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2123
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.640625,
+      "epoch": 3.3984,
+      "grad_norm": 0.022735716336833574,
+      "kl": 0.10400390625,
+      "learning_rate": 7.168e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2124
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.984375,
+      "epoch": 3.4,
+      "grad_norm": 0.10561759874409264,
+      "kl": 0.109619140625,
+      "learning_rate": 7.166666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2125
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.296875,
+      "epoch": 3.4016,
+      "grad_norm": 0.005425110282688084,
+      "kl": 0.10302734375,
+      "learning_rate": 7.165333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2126
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.109375,
+      "epoch": 3.4032,
+      "grad_norm": 0.0061104514557321675,
+      "kl": 0.10009765625,
+      "learning_rate": 7.164e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2127
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.1875,
+      "epoch": 3.4048,
+      "grad_norm": 0.005263419272828468,
+      "kl": 0.0986328125,
+      "learning_rate": 7.162666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2128
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.984375,
+      "epoch": 3.4064,
+      "grad_norm": 0.006417856028452163,
+      "kl": 0.107666015625,
+      "learning_rate": 7.161333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2129
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.640625,
+      "epoch": 3.408,
+      "grad_norm": 0.005836614428577645,
+      "kl": 0.1025390625,
+      "learning_rate": 7.159999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2130
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.828125,
+      "epoch": 3.4096,
+      "grad_norm": 0.0050736579729165,
+      "kl": 0.101806640625,
+      "learning_rate": 7.158666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2131
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.09375,
+      "epoch": 3.4112,
+      "grad_norm": 0.004689324062632128,
+      "kl": 0.107421875,
+      "learning_rate": 7.157333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2132
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.703125,
+      "epoch": 3.4128,
+      "grad_norm": 0.005676339337847811,
+      "kl": 0.10205078125,
+      "learning_rate": 7.156e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2133
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.203125,
+      "epoch": 3.4144,
+      "grad_norm": 0.006374089348806632,
+      "kl": 0.099365234375,
+      "learning_rate": 7.154666666666667e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 2134
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.765625,
+      "epoch": 3.416,
+      "grad_norm": 0.0070227830302423975,
+      "kl": 0.09814453125,
+      "learning_rate": 7.153333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2135
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.09375,
+      "epoch": 3.4176,
+      "grad_norm": 0.006242170870236814,
+      "kl": 0.10791015625,
+      "learning_rate": 7.151999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2136
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.390625,
+      "epoch": 3.4192,
+      "grad_norm": 30.343486708721525,
+      "kl": 0.36572265625,
+      "learning_rate": 7.150666666666666e-07,
+      "loss": 0.0004,
+      "reward": 1.515625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.515625,
+      "step": 2137
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 141.46875,
+      "epoch": 3.4208,
+      "grad_norm": 0.0045258539941825686,
+      "kl": 0.102783203125,
+      "learning_rate": 7.149333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2138
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.671875,
+      "epoch": 3.4224,
+      "grad_norm": 0.01714265290350637,
+      "kl": 0.0986328125,
+      "learning_rate": 7.147999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2139
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.203125,
+      "epoch": 3.424,
+      "grad_norm": 0.007676589514563665,
+      "kl": 0.094482421875,
+      "learning_rate": 7.146666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2140
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.1875,
+      "epoch": 3.4256,
+      "grad_norm": 0.006158799242659314,
+      "kl": 0.0986328125,
+      "learning_rate": 7.145333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2141
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.625,
+      "epoch": 3.4272,
+      "grad_norm": 0.0043270530190881226,
+      "kl": 0.098876953125,
+      "learning_rate": 7.144e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2142
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.375,
+      "epoch": 3.4288,
+      "grad_norm": 0.0052866915294885975,
+      "kl": 0.1015625,
+      "learning_rate": 7.142666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2143
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.0625,
+      "epoch": 3.4304,
+      "grad_norm": 0.00427266580175989,
+      "kl": 0.093994140625,
+      "learning_rate": 7.141333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2144
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.96875,
+      "epoch": 3.432,
+      "grad_norm": 0.00473472686797937,
+      "kl": 0.103515625,
+      "learning_rate": 7.14e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2145
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.34375,
+      "epoch": 3.4336,
+      "grad_norm": 0.015243496081891959,
+      "kl": 0.094970703125,
+      "learning_rate": 7.138666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2146
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.953125,
+      "epoch": 3.4352,
+      "grad_norm": 0.00496968513772915,
+      "kl": 0.0966796875,
+      "learning_rate": 7.137333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2147
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.703125,
+      "epoch": 3.4368,
+      "grad_norm": 12.762674484865162,
+      "kl": 0.092529296875,
+      "learning_rate": 7.135999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.8125,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.8125,
+      "step": 2148
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.0625,
+      "epoch": 3.4384,
+      "grad_norm": 0.006743543989878013,
+      "kl": 0.098876953125,
+      "learning_rate": 7.134666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2149
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.90625,
+      "epoch": 3.44,
+      "grad_norm": 0.004299801319300519,
+      "kl": 0.096923828125,
+      "learning_rate": 7.133333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2150
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.78125,
+      "epoch": 3.4416,
+      "grad_norm": 0.004349388346914053,
+      "kl": 0.09619140625,
+      "learning_rate": 7.131999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2151
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.140625,
+      "epoch": 3.4432,
+      "grad_norm": 0.009241873681125811,
+      "kl": 0.1162109375,
+      "learning_rate": 7.130666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2152
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.609375,
+      "epoch": 3.4448,
+      "grad_norm": 0.0073277055927595865,
+      "kl": 0.09033203125,
+      "learning_rate": 7.129333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2153
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.671875,
+      "epoch": 3.4464,
+      "grad_norm": 0.009154510829302288,
+      "kl": 0.100830078125,
+      "learning_rate": 7.128e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2154
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.3125,
+      "epoch": 3.448,
+      "grad_norm": 0.004467637805688453,
+      "kl": 0.094970703125,
+      "learning_rate": 7.126666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2155
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.625,
+      "epoch": 3.4496,
+      "grad_norm": 0.004969211214365444,
+      "kl": 0.091064453125,
+      "learning_rate": 7.125333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2156
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 142.546875,
+      "epoch": 3.4512,
+      "grad_norm": 0.005628303791031671,
+      "kl": 0.097412109375,
+      "learning_rate": 7.124e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2157
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 145.640625,
+      "epoch": 3.4528,
+      "grad_norm": 0.007006695317638204,
+      "kl": 0.0966796875,
+      "learning_rate": 7.122666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2158
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.75,
+      "epoch": 3.4544,
+      "grad_norm": 0.003790851872146526,
+      "kl": 0.092529296875,
+      "learning_rate": 7.121333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2159
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.859375,
+      "epoch": 3.456,
+      "grad_norm": 0.005117308364486238,
+      "kl": 0.101318359375,
+      "learning_rate": 7.119999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2160
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.328125,
+      "epoch": 3.4576000000000002,
+      "grad_norm": 5.062116973388454,
+      "kl": 0.087890625,
+      "learning_rate": 7.118666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.8125,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.8125,
+      "step": 2161
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.15625,
+      "epoch": 3.4592,
+      "grad_norm": 0.0057341937655805255,
+      "kl": 0.096435546875,
+      "learning_rate": 7.117333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2162
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 143.671875,
+      "epoch": 3.4608,
+      "grad_norm": 0.004003217399308068,
+      "kl": 0.089111328125,
+      "learning_rate": 7.116e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2163
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.953125,
+      "epoch": 3.4624,
+      "grad_norm": 0.006672387596445885,
+      "kl": 0.087890625,
+      "learning_rate": 7.114666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2164
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.171875,
+      "epoch": 3.464,
+      "grad_norm": 0.006387092127479219,
+      "kl": 0.099609375,
+      "learning_rate": 7.113333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2165
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.90625,
+      "epoch": 3.4656000000000002,
+      "grad_norm": 0.004784045162560305,
+      "kl": 0.091552734375,
+      "learning_rate": 7.112000000000001e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2166
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 141.046875,
+      "epoch": 3.4672,
+      "grad_norm": 18.54677194065343,
+      "kl": 0.092041015625,
+      "learning_rate": 7.110666666666665e-07,
+      "loss": 0.0001,
+      "reward": 1.671875,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.671875,
+      "step": 2167
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.0625,
+      "epoch": 3.4688,
+      "grad_norm": 0.0060251944414971255,
+      "kl": 0.091796875,
+      "learning_rate": 7.109333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2168
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 147.28125,
+      "epoch": 3.4704,
+      "grad_norm": 0.004985654900587413,
+      "kl": 0.095947265625,
+      "learning_rate": 7.107999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2169
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.5,
+      "epoch": 3.472,
+      "grad_norm": 0.004406390139461559,
+      "kl": 0.09375,
+      "learning_rate": 7.106666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2170
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.1875,
+      "epoch": 3.4736000000000002,
+      "grad_norm": 0.004605347084656732,
+      "kl": 0.08935546875,
+      "learning_rate": 7.105333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2171
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 143.71875,
+      "epoch": 3.4752,
+      "grad_norm": 0.004158104350641111,
+      "kl": 0.090087890625,
+      "learning_rate": 7.104e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2172
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.078125,
+      "epoch": 3.4768,
+      "grad_norm": 0.006978608786955054,
+      "kl": 0.095703125,
+      "learning_rate": 7.102666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2173
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 143.375,
+      "epoch": 3.4784,
+      "grad_norm": 0.007499461624151407,
+      "kl": 0.100830078125,
+      "learning_rate": 7.101333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2174
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.375,
+      "epoch": 3.48,
+      "grad_norm": 0.00560335195533077,
+      "kl": 0.094482421875,
+      "learning_rate": 7.1e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2175
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.421875,
+      "epoch": 3.4816,
+      "grad_norm": 0.010166112180799191,
+      "kl": 0.090087890625,
+      "learning_rate": 7.098666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 2176
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.109375,
+      "epoch": 3.4832,
+      "grad_norm": 0.006665651960290914,
+      "kl": 0.092529296875,
+      "learning_rate": 7.097333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2177
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 145.6875,
+      "epoch": 3.4848,
+      "grad_norm": 0.004987671913729233,
+      "kl": 0.09130859375,
+      "learning_rate": 7.096e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2178
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.546875,
+      "epoch": 3.4864,
+      "grad_norm": 0.0070690055616673695,
+      "kl": 0.101318359375,
+      "learning_rate": 7.094666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2179
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.984375,
+      "epoch": 3.488,
+      "grad_norm": 0.013248987535546491,
+      "kl": 0.090576171875,
+      "learning_rate": 7.093333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2180
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.578125,
+      "epoch": 3.4896,
+      "grad_norm": 0.007381138169440605,
+      "kl": 0.08984375,
+      "learning_rate": 7.092e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2181
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.40625,
+      "epoch": 3.4912,
+      "grad_norm": 0.004711502845511917,
+      "kl": 0.093994140625,
+      "learning_rate": 7.090666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2182
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 141.9375,
+      "epoch": 3.4928,
+      "grad_norm": 0.005085327112530645,
+      "kl": 0.09423828125,
+      "learning_rate": 7.089333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2183
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.28125,
+      "epoch": 3.4944,
+      "grad_norm": 0.0041830908279225365,
+      "kl": 0.090087890625,
+      "learning_rate": 7.088e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2184
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 143.71875,
+      "epoch": 3.496,
+      "grad_norm": 45.16866044609127,
+      "kl": 0.090576171875,
+      "learning_rate": 7.086666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.703125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.703125,
+      "step": 2185
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.0625,
+      "epoch": 3.4976,
+      "grad_norm": 0.0052949283498400345,
+      "kl": 0.09228515625,
+      "learning_rate": 7.085333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2186
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.578125,
+      "epoch": 3.4992,
+      "grad_norm": 0.005795613888004041,
+      "kl": 0.09765625,
+      "learning_rate": 7.084e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2187
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.265625,
+      "epoch": 3.5008,
+      "grad_norm": 0.005691093005670626,
+      "kl": 0.08740234375,
+      "learning_rate": 7.082666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2188
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 151.359375,
+      "epoch": 3.5023999999999997,
+      "grad_norm": 0.004303748063489153,
+      "kl": 0.091064453125,
+      "learning_rate": 7.081333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2189
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 144.140625,
+      "epoch": 3.504,
+      "grad_norm": 2.7857822016301874,
+      "kl": 0.089111328125,
+      "learning_rate": 7.079999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 2190
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.53125,
+      "epoch": 3.5056000000000003,
+      "grad_norm": 0.0061806764653001695,
+      "kl": 0.08984375,
+      "learning_rate": 7.078666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2191
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.59375,
+      "epoch": 3.5072,
+      "grad_norm": 0.00527155608761704,
+      "kl": 0.093017578125,
+      "learning_rate": 7.077333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2192
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.171875,
+      "epoch": 3.5088,
+      "grad_norm": 0.0036659006151627182,
+      "kl": 0.0966796875,
+      "learning_rate": 7.076e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2193
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.875,
+      "epoch": 3.5103999999999997,
+      "grad_norm": 0.004579928628174406,
+      "kl": 0.087890625,
+      "learning_rate": 7.074666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2194
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 143.1875,
+      "epoch": 3.512,
+      "grad_norm": 0.004155622965052897,
+      "kl": 0.086669921875,
+      "learning_rate": 7.073333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2195
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.609375,
+      "epoch": 3.5136,
+      "grad_norm": 9.076445487708805,
+      "kl": 0.09130859375,
+      "learning_rate": 7.072e-07,
+      "loss": 0.0001,
+      "reward": 1.265625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.265625,
+      "step": 2196
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.046875,
+      "epoch": 3.5152,
+      "grad_norm": 8.686842559621216,
+      "kl": 0.088623046875,
+      "learning_rate": 7.070666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.515625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.515625,
+      "step": 2197
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 144.65625,
+      "epoch": 3.5168,
+      "grad_norm": 0.00837513768981975,
+      "kl": 0.082275390625,
+      "learning_rate": 7.069333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 2198
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 147.421875,
+      "epoch": 3.5183999999999997,
+      "grad_norm": 7.844719456628962,
+      "kl": 0.10205078125,
+      "learning_rate": 7.068e-07,
+      "loss": 0.0001,
+      "reward": 1.640625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.640625,
+      "step": 2199
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.125,
+      "epoch": 3.52,
+      "grad_norm": 6.8593215108388454,
+      "kl": 0.079833984375,
+      "learning_rate": 7.066666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.609375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.609375,
+      "step": 2200
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.96875,
+      "epoch": 3.5216,
+      "grad_norm": 9.971224012624955,
+      "kl": 0.084228515625,
+      "learning_rate": 7.065333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.53125,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.53125,
+      "step": 2201
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 144.296875,
+      "epoch": 3.5232,
+      "grad_norm": 0.008070346985854696,
+      "kl": 0.08935546875,
+      "learning_rate": 7.064e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2202
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 144.125,
+      "epoch": 3.5248,
+      "grad_norm": 0.006004599294501488,
+      "kl": 0.08056640625,
+      "learning_rate": 7.062666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2203
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.5625,
+      "epoch": 3.5263999999999998,
+      "grad_norm": 0.004967606792249327,
+      "kl": 0.08203125,
+      "learning_rate": 7.061333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2204
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 149.03125,
+      "epoch": 3.528,
+      "grad_norm": 0.006047433976184479,
+      "kl": 0.08740234375,
+      "learning_rate": 7.059999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2205
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 141.71875,
+      "epoch": 3.5296,
+      "grad_norm": 8.382463870412835,
+      "kl": 0.0849609375,
+      "learning_rate": 7.058666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.703125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.703125,
+      "step": 2206
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 142.015625,
+      "epoch": 3.5312,
+      "grad_norm": 7.040151098907058,
+      "kl": 0.089111328125,
+      "learning_rate": 7.057333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.59375,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.59375,
+      "step": 2207
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.96875,
+      "epoch": 3.5328,
+      "grad_norm": 0.005334029621870948,
+      "kl": 0.0869140625,
+      "learning_rate": 7.056e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2208
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 148.703125,
+      "epoch": 3.5343999999999998,
+      "grad_norm": 0.004071085853113224,
+      "kl": 0.080322265625,
+      "learning_rate": 7.054666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2209
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 142.796875,
+      "epoch": 3.536,
+      "grad_norm": 5.4448867018977705,
+      "kl": 0.094482421875,
+      "learning_rate": 7.053333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.578125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.578125,
+      "step": 2210
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.3125,
+      "epoch": 3.5376,
+      "grad_norm": 0.010511215245028687,
+      "kl": 0.0888671875,
+      "learning_rate": 7.052e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2211
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.828125,
+      "epoch": 3.5392,
+      "grad_norm": 0.0062539080091507954,
+      "kl": 0.084716796875,
+      "learning_rate": 7.050666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2212
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 142.75,
+      "epoch": 3.5408,
+      "grad_norm": 0.005831978334246557,
+      "kl": 0.092529296875,
+      "learning_rate": 7.049333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2213
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 144.0625,
+      "epoch": 3.5423999999999998,
+      "grad_norm": 0.004887544462047133,
+      "kl": 0.097412109375,
+      "learning_rate": 7.047999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2214
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.140625,
+      "epoch": 3.544,
+      "grad_norm": 0.005930029797896277,
+      "kl": 0.087890625,
+      "learning_rate": 7.046666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2215
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.609375,
+      "epoch": 3.5456,
+      "grad_norm": 0.004006028726565302,
+      "kl": 0.0859375,
+      "learning_rate": 7.045333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 2216
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 146.59375,
+      "epoch": 3.5472,
+      "grad_norm": 0.00836570693301341,
+      "kl": 0.098388671875,
+      "learning_rate": 7.044e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2217
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 148.703125,
+      "epoch": 3.5488,
+      "grad_norm": 0.005139946591678097,
+      "kl": 0.093017578125,
+      "learning_rate": 7.042666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2218
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.953125,
+      "epoch": 3.5504,
+      "grad_norm": 136.35964066723866,
+      "kl": 0.0888671875,
+      "learning_rate": 7.041333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.703125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.703125,
+      "step": 2219
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 144.640625,
+      "epoch": 3.552,
+      "grad_norm": 0.041942767440479485,
+      "kl": 0.092529296875,
+      "learning_rate": 7.04e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2220
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.171875,
+      "epoch": 3.5536,
+      "grad_norm": 0.00680785521091184,
+      "kl": 0.100341796875,
+      "learning_rate": 7.038666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2221
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 144.953125,
+      "epoch": 3.5552,
+      "grad_norm": 3.9095941912783108,
+      "kl": 0.09228515625,
+      "learning_rate": 7.037333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.921875,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.921875,
+      "step": 2222
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 143.90625,
+      "epoch": 3.5568,
+      "grad_norm": 0.00636562387200427,
+      "kl": 0.094970703125,
+      "learning_rate": 7.035999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.25,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.25,
+      "step": 2223
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.546875,
+      "epoch": 3.5584,
+      "grad_norm": 0.004278283898773982,
+      "kl": 0.090087890625,
+      "learning_rate": 7.034666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2224
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.34375,
+      "epoch": 3.56,
+      "grad_norm": 0.0037684174962612565,
+      "kl": 0.088134765625,
+      "learning_rate": 7.033333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2225
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 143.0625,
+      "epoch": 3.5616,
+      "grad_norm": 0.003968551572731447,
+      "kl": 0.087890625,
+      "learning_rate": 7.032e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2226
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 144.875,
+      "epoch": 3.5632,
+      "grad_norm": 0.004593492315584341,
+      "kl": 0.085693359375,
+      "learning_rate": 7.030666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2227
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 148.296875,
+      "epoch": 3.5648,
+      "grad_norm": 0.004273732520404633,
+      "kl": 0.086181640625,
+      "learning_rate": 7.029333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2228
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.578125,
+      "epoch": 3.5664,
+      "grad_norm": 0.005506830132555107,
+      "kl": 0.097900390625,
+      "learning_rate": 7.028e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2229
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.625,
+      "epoch": 3.568,
+      "grad_norm": 6.676618505442662,
+      "kl": 0.096435546875,
+      "learning_rate": 7.026666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.71875,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.71875,
+      "step": 2230
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 145.0625,
+      "epoch": 3.5696,
+      "grad_norm": 0.0049291973240150845,
+      "kl": 0.085205078125,
+      "learning_rate": 7.025333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2231
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 145.421875,
+      "epoch": 3.5712,
+      "grad_norm": 0.006397576959716904,
+      "kl": 0.088134765625,
+      "learning_rate": 7.024e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2232
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 149.140625,
+      "epoch": 3.5728,
+      "grad_norm": 0.0060212273020160845,
+      "kl": 0.093017578125,
+      "learning_rate": 7.022666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2233
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 143.046875,
+      "epoch": 3.5744,
+      "grad_norm": 6.889076886608771,
+      "kl": 0.100341796875,
+      "learning_rate": 7.021333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.453125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.453125,
+      "step": 2234
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 151.140625,
+      "epoch": 3.576,
+      "grad_norm": 0.004509588387682993,
+      "kl": 0.087158203125,
+      "learning_rate": 7.019999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2235
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 147.828125,
+      "epoch": 3.5776,
+      "grad_norm": 0.004601585755340759,
+      "kl": 0.09130859375,
+      "learning_rate": 7.018666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2236
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 146.328125,
+      "epoch": 3.5792,
+      "grad_norm": 0.005162808734956054,
+      "kl": 0.090087890625,
+      "learning_rate": 7.017333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2237
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 146.296875,
+      "epoch": 3.5808,
+      "grad_norm": 0.004559724949262744,
+      "kl": 0.085693359375,
+      "learning_rate": 7.016e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2238
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.5,
+      "epoch": 3.5824,
+      "grad_norm": 17.478039531429204,
+      "kl": 0.091796875,
+      "learning_rate": 7.014666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.65625,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.65625,
+      "step": 2239
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 146.09375,
+      "epoch": 3.584,
+      "grad_norm": 0.00979566769292862,
+      "kl": 0.095947265625,
+      "learning_rate": 7.013333333333334e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 2240
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 148.828125,
+      "epoch": 3.5856,
+      "grad_norm": 16.370605491061582,
+      "kl": 0.090576171875,
+      "learning_rate": 7.012000000000001e-07,
+      "loss": 0.0001,
+      "reward": 1.703125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.703125,
+      "step": 2241
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 149.265625,
+      "epoch": 3.5872,
+      "grad_norm": 0.004302425254413143,
+      "kl": 0.082763671875,
+      "learning_rate": 7.010666666666665e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2242
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 149.34375,
+      "epoch": 3.5888,
+      "grad_norm": 0.005491403761470396,
+      "kl": 0.093017578125,
+      "learning_rate": 7.009333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2243
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 152.453125,
+      "epoch": 3.5904,
+      "grad_norm": 0.00576699120069471,
+      "kl": 0.095703125,
+      "learning_rate": 7.007999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2244
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 148.0625,
+      "epoch": 3.592,
+      "grad_norm": 4.166748638473592,
+      "kl": 0.086669921875,
+      "learning_rate": 7.006666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.265625,
+      "reward_std": 0.10205793008208275,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.28125,
+      "step": 2245
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 141.5625,
+      "epoch": 3.5936,
+      "grad_norm": 0.004352490328597202,
+      "kl": 0.0859375,
+      "learning_rate": 7.005333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2246
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 150.265625,
+      "epoch": 3.5952,
+      "grad_norm": 0.005598427942070745,
+      "kl": 0.090576171875,
+      "learning_rate": 7.004e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2247
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 150.875,
+      "epoch": 3.5968,
+      "grad_norm": 5.5158621995149355,
+      "kl": 0.0908203125,
+      "learning_rate": 7.002666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 2248
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 146.453125,
+      "epoch": 3.5984,
+      "grad_norm": 0.007712122894116483,
+      "kl": 0.09228515625,
+      "learning_rate": 7.001333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2249
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 146.40625,
+      "epoch": 3.6,
+      "grad_norm": 0.0045211982166701616,
+      "kl": 0.097900390625,
+      "learning_rate": 7e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2250
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 146.96875,
+      "epoch": 3.6016,
+      "grad_norm": 0.011499848066526668,
+      "kl": 0.09423828125,
+      "learning_rate": 6.998666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2251
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 143.25,
+      "epoch": 3.6032,
+      "grad_norm": 0.004935544852644549,
+      "kl": 0.0888671875,
+      "learning_rate": 6.997333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2252
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 146.515625,
+      "epoch": 3.6048,
+      "grad_norm": 8.26354145610504,
+      "kl": 0.086669921875,
+      "learning_rate": 6.995999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.703125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.703125,
+      "step": 2253
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 144.109375,
+      "epoch": 3.6064,
+      "grad_norm": 0.008611350622632942,
+      "kl": 0.085205078125,
+      "learning_rate": 6.994666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2254
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 142.65625,
+      "epoch": 3.608,
+      "grad_norm": 0.008459035100675472,
+      "kl": 0.085693359375,
+      "learning_rate": 6.993333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2255
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 144.265625,
+      "epoch": 3.6096,
+      "grad_norm": 0.004705358141757878,
+      "kl": 0.084716796875,
+      "learning_rate": 6.992e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2256
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 144.25,
+      "epoch": 3.6112,
+      "grad_norm": 0.004293985388661945,
+      "kl": 0.08544921875,
+      "learning_rate": 6.990666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2257
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.546875,
+      "epoch": 3.6128,
+      "grad_norm": 0.009765762239364545,
+      "kl": 0.09130859375,
+      "learning_rate": 6.989333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2258
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.59375,
+      "epoch": 3.6144,
+      "grad_norm": 9.37924469338974,
+      "kl": 0.078857421875,
+      "learning_rate": 6.988e-07,
+      "loss": 0.0001,
+      "reward": 1.5625,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5625,
+      "step": 2259
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 142.375,
+      "epoch": 3.616,
+      "grad_norm": 0.003901877327275357,
+      "kl": 0.080078125,
+      "learning_rate": 6.986666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2260
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 146.5,
+      "epoch": 3.6176,
+      "grad_norm": 2.828478172065943,
+      "kl": 0.08935546875,
+      "learning_rate": 6.985333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.609375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.609375,
+      "step": 2261
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.265625,
+      "epoch": 3.6192,
+      "grad_norm": 0.004899979040110671,
+      "kl": 0.08349609375,
+      "learning_rate": 6.984e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2262
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 147.3125,
+      "epoch": 3.6208,
+      "grad_norm": 0.006503593765043026,
+      "kl": 0.091796875,
+      "learning_rate": 6.982666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2263
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.078125,
+      "epoch": 3.6224,
+      "grad_norm": 7.262600691913564,
+      "kl": 0.087646484375,
+      "learning_rate": 6.981333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.4375,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.4375,
+      "step": 2264
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 146.484375,
+      "epoch": 3.624,
+      "grad_norm": 0.004323629939022822,
+      "kl": 0.088623046875,
+      "learning_rate": 6.979999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2265
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.828125,
+      "epoch": 3.6256,
+      "grad_norm": 0.004964848135784546,
+      "kl": 0.0859375,
+      "learning_rate": 6.978666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2266
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 142.125,
+      "epoch": 3.6272,
+      "grad_norm": 0.0042981902856145835,
+      "kl": 0.087646484375,
+      "learning_rate": 6.977333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2267
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.453125,
+      "epoch": 3.6288,
+      "grad_norm": 0.00497285396385034,
+      "kl": 0.084716796875,
+      "learning_rate": 6.976e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2268
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 141.5625,
+      "epoch": 3.6304,
+      "grad_norm": 5.424212586212769,
+      "kl": 0.09326171875,
+      "learning_rate": 6.974666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.640625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.640625,
+      "step": 2269
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.875,
+      "epoch": 3.632,
+      "grad_norm": 0.010428465440061579,
+      "kl": 0.0947265625,
+      "learning_rate": 6.973333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2270
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.59375,
+      "epoch": 3.6336,
+      "grad_norm": 0.004004545502187796,
+      "kl": 0.09228515625,
+      "learning_rate": 6.972e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2271
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.671875,
+      "epoch": 3.6352,
+      "grad_norm": 0.004651883825537912,
+      "kl": 0.0869140625,
+      "learning_rate": 6.970666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2272
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 141.734375,
+      "epoch": 3.6368,
+      "grad_norm": 0.017307403698764986,
+      "kl": 0.093017578125,
+      "learning_rate": 6.969333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2273
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.265625,
+      "epoch": 3.6384,
+      "grad_norm": 0.009279666247560107,
+      "kl": 0.09033203125,
+      "learning_rate": 6.967999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2274
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.234375,
+      "epoch": 3.64,
+      "grad_norm": 0.011084031863762856,
+      "kl": 0.08642578125,
+      "learning_rate": 6.966666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2275
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.46875,
+      "epoch": 3.6416,
+      "grad_norm": 0.0049205730674631775,
+      "kl": 0.083251953125,
+      "learning_rate": 6.965333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2276
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.3125,
+      "epoch": 3.6432,
+      "grad_norm": 0.005439242125681791,
+      "kl": 0.087890625,
+      "learning_rate": 6.964e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2277
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.609375,
+      "epoch": 3.6448,
+      "grad_norm": 0.004728140600487053,
+      "kl": 0.083984375,
+      "learning_rate": 6.962666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2278
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.421875,
+      "epoch": 3.6464,
+      "grad_norm": 12.84287978397286,
+      "kl": 0.094482421875,
+      "learning_rate": 6.961333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.671875,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.671875,
+      "step": 2279
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.359375,
+      "epoch": 3.648,
+      "grad_norm": 0.005811776329936363,
+      "kl": 0.0966796875,
+      "learning_rate": 6.959999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2280
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.671875,
+      "epoch": 3.6496,
+      "grad_norm": 0.005177968184248439,
+      "kl": 0.087646484375,
+      "learning_rate": 6.958666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2281
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.5625,
+      "epoch": 3.6512000000000002,
+      "grad_norm": 0.022683648274495375,
+      "kl": 0.0888671875,
+      "learning_rate": 6.957333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2282
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.453125,
+      "epoch": 3.6528,
+      "grad_norm": 9.911170611259044,
+      "kl": 0.08154296875,
+      "learning_rate": 6.956e-07,
+      "loss": 0.0001,
+      "reward": 1.703125,
+      "reward_std": 0.11100947856903076,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.703125,
+      "step": 2283
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.609375,
+      "epoch": 3.6544,
+      "grad_norm": 0.007715900796256404,
+      "kl": 0.0859375,
+      "learning_rate": 6.954666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2284
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.671875,
+      "epoch": 3.656,
+      "grad_norm": 0.004451759911212696,
+      "kl": 0.082763671875,
+      "learning_rate": 6.953333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2285
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.671875,
+      "epoch": 3.6576,
+      "grad_norm": 6.758967644485185,
+      "kl": 0.09521484375,
+      "learning_rate": 6.952e-07,
+      "loss": 0.0001,
+      "reward": 1.578125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.578125,
+      "step": 2286
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.015625,
+      "epoch": 3.6592000000000002,
+      "grad_norm": 0.008832034626515611,
+      "kl": 0.091796875,
+      "learning_rate": 6.950666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2287
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.515625,
+      "epoch": 3.6608,
+      "grad_norm": 6.325049820246651,
+      "kl": 0.09375,
+      "learning_rate": 6.949333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.484375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.484375,
+      "step": 2288
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.78125,
+      "epoch": 3.6624,
+      "grad_norm": 12.212489966421176,
+      "kl": 0.087646484375,
+      "learning_rate": 6.947999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.890625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.890625,
+      "step": 2289
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.28125,
+      "epoch": 3.664,
+      "grad_norm": 0.0048484929008876234,
+      "kl": 0.085205078125,
+      "learning_rate": 6.946666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2290
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.265625,
+      "epoch": 3.6656,
+      "grad_norm": 1.3403753550834536,
+      "kl": 0.107421875,
+      "learning_rate": 6.945333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2291
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.1875,
+      "epoch": 3.6672000000000002,
+      "grad_norm": 0.005765024554249471,
+      "kl": 0.08837890625,
+      "learning_rate": 6.944e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 2292
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.4375,
+      "epoch": 3.6688,
+      "grad_norm": 0.004696969685058984,
+      "kl": 0.090576171875,
+      "learning_rate": 6.942666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2293
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.0625,
+      "epoch": 3.6704,
+      "grad_norm": 0.0059357239913590334,
+      "kl": 0.090576171875,
+      "learning_rate": 6.941333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 2294
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.359375,
+      "epoch": 3.672,
+      "grad_norm": 0.005319199917124415,
+      "kl": 0.09619140625,
+      "learning_rate": 6.939999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2295
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.125,
+      "epoch": 3.6736,
+      "grad_norm": 0.00517195786644057,
+      "kl": 0.09423828125,
+      "learning_rate": 6.938666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2296
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.40625,
+      "epoch": 3.6752000000000002,
+      "grad_norm": 0.0061856673455487725,
+      "kl": 0.08447265625,
+      "learning_rate": 6.937333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2297
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.5,
+      "epoch": 3.6768,
+      "grad_norm": 0.005343259662461003,
+      "kl": 0.094970703125,
+      "learning_rate": 6.935999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2298
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.6875,
+      "epoch": 3.6784,
+      "grad_norm": 0.005282845933290416,
+      "kl": 0.091064453125,
+      "learning_rate": 6.934666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 2299
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.125,
+      "epoch": 3.68,
+      "grad_norm": 0.005732301919100361,
+      "kl": 0.092041015625,
+      "learning_rate": 6.933333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2300
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.953125,
+      "epoch": 3.6816,
+      "grad_norm": 0.00529987576239362,
+      "kl": 0.09326171875,
+      "learning_rate": 6.932e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2301
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.6875,
+      "epoch": 3.6832000000000003,
+      "grad_norm": 0.0071737992096234295,
+      "kl": 0.094970703125,
+      "learning_rate": 6.930666666666667e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 2302
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.328125,
+      "epoch": 3.6848,
+      "grad_norm": 9.659560753563937,
+      "kl": 0.09423828125,
+      "learning_rate": 6.929333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.390625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.390625,
+      "step": 2303
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.171875,
+      "epoch": 3.6864,
+      "grad_norm": 1.5793895319259181,
+      "kl": 0.09375,
+      "learning_rate": 6.928e-07,
+      "loss": 0.0001,
+      "reward": 1.296875,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.296875,
+      "step": 2304
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.890625,
+      "epoch": 3.6879999999999997,
+      "grad_norm": 0.01650739999101953,
+      "kl": 0.088134765625,
+      "learning_rate": 6.926666666666666e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 2305
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.203125,
+      "epoch": 3.6896,
+      "grad_norm": 0.00596997618534019,
+      "kl": 0.0888671875,
+      "learning_rate": 6.925333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2306
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.359375,
+      "epoch": 3.6912000000000003,
+      "grad_norm": 4.196746189263735,
+      "kl": 0.09033203125,
+      "learning_rate": 6.924e-07,
+      "loss": 0.0001,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 2307
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.140625,
+      "epoch": 3.6928,
+      "grad_norm": 0.005279488231546182,
+      "kl": 0.099365234375,
+      "learning_rate": 6.922666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 2308
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.625,
+      "epoch": 3.6944,
+      "grad_norm": 0.010289893415169683,
+      "kl": 0.091064453125,
+      "learning_rate": 6.921333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2309
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.421875,
+      "epoch": 3.6959999999999997,
+      "grad_norm": 0.006308153142484358,
+      "kl": 0.098876953125,
+      "learning_rate": 6.919999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2310
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.453125,
+      "epoch": 3.6976,
+      "grad_norm": 4.576585406329555,
+      "kl": 0.0927734375,
+      "learning_rate": 6.918666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 2311
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.734375,
+      "epoch": 3.6992000000000003,
+      "grad_norm": 0.006304973545760664,
+      "kl": 0.099365234375,
+      "learning_rate": 6.917333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2312
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.65625,
+      "epoch": 3.7008,
+      "grad_norm": 8.085885169265703,
+      "kl": 0.10546875,
+      "learning_rate": 6.916e-07,
+      "loss": 0.0001,
+      "reward": 1.84375,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.84375,
+      "step": 2313
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.984375,
+      "epoch": 3.7024,
+      "grad_norm": 0.005863559862652111,
+      "kl": 0.099609375,
+      "learning_rate": 6.914666666666667e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 2314
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.953125,
+      "epoch": 3.7039999999999997,
+      "grad_norm": 0.035342641184584245,
+      "kl": 0.105224609375,
+      "learning_rate": 6.913333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 2315
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.828125,
+      "epoch": 3.7056,
+      "grad_norm": 0.005405298721914137,
+      "kl": 0.095703125,
+      "learning_rate": 6.912e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2316
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.359375,
+      "epoch": 3.7072000000000003,
+      "grad_norm": 0.005489504479634635,
+      "kl": 0.092529296875,
+      "learning_rate": 6.910666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2317
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.3125,
+      "epoch": 3.7088,
+      "grad_norm": 55.69429429017464,
+      "kl": 0.0986328125,
+      "learning_rate": 6.909333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.578125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.578125,
+      "step": 2318
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.609375,
+      "epoch": 3.7104,
+      "grad_norm": 0.006064137065175656,
+      "kl": 0.09326171875,
+      "learning_rate": 6.907999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 2319
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.453125,
+      "epoch": 3.7119999999999997,
+      "grad_norm": 0.0078003743539938495,
+      "kl": 0.10400390625,
+      "learning_rate": 6.906666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 2320
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.15625,
+      "epoch": 3.7136,
+      "grad_norm": 0.0065975372012450515,
+      "kl": 0.101806640625,
+      "learning_rate": 6.905333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2321
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.0,
+      "epoch": 3.7152,
+      "grad_norm": 0.005846024602555722,
+      "kl": 0.10009765625,
+      "learning_rate": 6.904e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2322
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.4375,
+      "epoch": 3.7168,
+      "grad_norm": 1.8726893038966632,
+      "kl": 0.1005859375,
+      "learning_rate": 6.902666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.765625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.765625,
+      "step": 2323
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.796875,
+      "epoch": 3.7184,
+      "grad_norm": 0.006237166238094651,
+      "kl": 0.09814453125,
+      "learning_rate": 6.901333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2324
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 111.703125,
+      "epoch": 3.7199999999999998,
+      "grad_norm": 0.0065827561392476286,
+      "kl": 0.09814453125,
+      "learning_rate": 6.9e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2325
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.875,
+      "epoch": 3.7216,
+      "grad_norm": 0.006053039941304552,
+      "kl": 0.103759765625,
+      "learning_rate": 6.898666666666665e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2326
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.484375,
+      "epoch": 3.7232,
+      "grad_norm": 0.006022583353365562,
+      "kl": 0.0986328125,
+      "learning_rate": 6.897333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2327
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.3125,
+      "epoch": 3.7248,
+      "grad_norm": 32.265140725922016,
+      "kl": 0.10791015625,
+      "learning_rate": 6.895999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.609375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.609375,
+      "step": 2328
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.75,
+      "epoch": 3.7264,
+      "grad_norm": 0.006457600322685193,
+      "kl": 0.112060546875,
+      "learning_rate": 6.894666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2329
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 115.796875,
+      "epoch": 3.7279999999999998,
+      "grad_norm": 0.005993493370428326,
+      "kl": 0.109130859375,
+      "learning_rate": 6.893333333333333e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 2330
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.875,
+      "epoch": 3.7296,
+      "grad_norm": 0.005771087362785205,
+      "kl": 0.098876953125,
+      "learning_rate": 6.892e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2331
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 116.640625,
+      "epoch": 3.7312,
+      "grad_norm": 0.006678206835464325,
+      "kl": 0.106689453125,
+      "learning_rate": 6.890666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2332
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.3125,
+      "epoch": 3.7328,
+      "grad_norm": 0.006832690163403729,
+      "kl": 0.109130859375,
+      "learning_rate": 6.889333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 2333
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.546875,
+      "epoch": 3.7344,
+      "grad_norm": 21.764566930798267,
+      "kl": 0.107421875,
+      "learning_rate": 6.888e-07,
+      "loss": 0.0001,
+      "reward": 1.84375,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.84375,
+      "step": 2334
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.140625,
+      "epoch": 3.7359999999999998,
+      "grad_norm": 0.008126309675013382,
+      "kl": 0.111083984375,
+      "learning_rate": 6.886666666666667e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 2335
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.765625,
+      "epoch": 3.7376,
+      "grad_norm": 54.18274266364752,
+      "kl": 0.1123046875,
+      "learning_rate": 6.885333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.421875,
+      "reward_std": 0.11100947856903076,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.4375,
+      "step": 2336
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.71875,
+      "epoch": 3.7392,
+      "grad_norm": 69.09637152054029,
+      "kl": 0.105224609375,
+      "learning_rate": 6.883999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.96875,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.96875,
+      "step": 2337
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.671875,
+      "epoch": 3.7408,
+      "grad_norm": 0.008377565559855556,
+      "kl": 0.119384765625,
+      "learning_rate": 6.882666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2338
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.921875,
+      "epoch": 3.7424,
+      "grad_norm": 27.429178299196398,
+      "kl": 0.1171875,
+      "learning_rate": 6.881333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.859375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.859375,
+      "step": 2339
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.796875,
+      "epoch": 3.7439999999999998,
+      "grad_norm": 0.00836251590191223,
+      "kl": 0.1181640625,
+      "learning_rate": 6.879999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2340
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.125,
+      "epoch": 3.7456,
+      "grad_norm": 0.006027754512225055,
+      "kl": 0.112060546875,
+      "learning_rate": 6.878666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2341
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.375,
+      "epoch": 3.7472,
+      "grad_norm": 0.010669165735504783,
+      "kl": 0.111328125,
+      "learning_rate": 6.877333333333333e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 2342
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.8125,
+      "epoch": 3.7488,
+      "grad_norm": 0.010197191530971027,
+      "kl": 0.109375,
+      "learning_rate": 6.876e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2343
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.578125,
+      "epoch": 3.7504,
+      "grad_norm": 0.01644357703419734,
+      "kl": 0.1123046875,
+      "learning_rate": 6.874666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2344
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.21875,
+      "epoch": 3.752,
+      "grad_norm": 0.005800007498389427,
+      "kl": 0.111572265625,
+      "learning_rate": 6.873333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2345
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 119.203125,
+      "epoch": 3.7536,
+      "grad_norm": 0.009269014838069535,
+      "kl": 0.109130859375,
+      "learning_rate": 6.872e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2346
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.875,
+      "epoch": 3.7552,
+      "grad_norm": 0.006117511835130651,
+      "kl": 0.109375,
+      "learning_rate": 6.870666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2347
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.359375,
+      "epoch": 3.7568,
+      "grad_norm": 0.0066006898062067896,
+      "kl": 0.104736328125,
+      "learning_rate": 6.869333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2348
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.6875,
+      "epoch": 3.7584,
+      "grad_norm": 0.011397588768612694,
+      "kl": 0.10791015625,
+      "learning_rate": 6.867999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2349
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.734375,
+      "epoch": 3.76,
+      "grad_norm": 0.008998018176457111,
+      "kl": 0.10546875,
+      "learning_rate": 6.866666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2350
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.375,
+      "epoch": 3.7616,
+      "grad_norm": 9.33189725054124,
+      "kl": 0.105224609375,
+      "learning_rate": 6.865333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.78125,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.78125,
+      "step": 2351
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.125,
+      "epoch": 3.7632,
+      "grad_norm": 0.010715305603647479,
+      "kl": 0.1005859375,
+      "learning_rate": 6.864e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2352
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.40625,
+      "epoch": 3.7648,
+      "grad_norm": 0.008362688981307707,
+      "kl": 0.10693359375,
+      "learning_rate": 6.862666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2353
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.234375,
+      "epoch": 3.7664,
+      "grad_norm": 0.006640789246448329,
+      "kl": 0.111083984375,
+      "learning_rate": 6.861333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2354
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.765625,
+      "epoch": 3.768,
+      "grad_norm": 0.007638844566135624,
+      "kl": 0.099853515625,
+      "learning_rate": 6.86e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 2355
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.96875,
+      "epoch": 3.7696,
+      "grad_norm": 0.008500908169736685,
+      "kl": 0.111083984375,
+      "learning_rate": 6.858666666666666e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 2356
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.0,
+      "epoch": 3.7712,
+      "grad_norm": 0.09900124870172604,
+      "kl": 0.114990234375,
+      "learning_rate": 6.857333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2357
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.296875,
+      "epoch": 3.7728,
+      "grad_norm": 0.007600208123354458,
+      "kl": 0.103515625,
+      "learning_rate": 6.855999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2358
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.046875,
+      "epoch": 3.7744,
+      "grad_norm": 0.031621504722721,
+      "kl": 0.10302734375,
+      "learning_rate": 6.854666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2359
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.109375,
+      "epoch": 3.776,
+      "grad_norm": 0.006911950537721606,
+      "kl": 0.119384765625,
+      "learning_rate": 6.853333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2360
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.734375,
+      "epoch": 3.7776,
+      "grad_norm": 37.192000462679914,
+      "kl": 0.11181640625,
+      "learning_rate": 6.852e-07,
+      "loss": 0.0001,
+      "reward": 1.78125,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.78125,
+      "step": 2361
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.296875,
+      "epoch": 3.7792,
+      "grad_norm": 0.015516089370425088,
+      "kl": 0.1103515625,
+      "learning_rate": 6.850666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2362
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.96875,
+      "epoch": 3.7808,
+      "grad_norm": 0.007023726665422061,
+      "kl": 0.113525390625,
+      "learning_rate": 6.849333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2363
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.90625,
+      "epoch": 3.7824,
+      "grad_norm": 15.343012214929727,
+      "kl": 0.09765625,
+      "learning_rate": 6.847999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.390625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.390625,
+      "step": 2364
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.828125,
+      "epoch": 3.784,
+      "grad_norm": 0.029804844984485875,
+      "kl": 0.099365234375,
+      "learning_rate": 6.846666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2365
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.8125,
+      "epoch": 3.7856,
+      "grad_norm": 0.008626998261770252,
+      "kl": 0.099365234375,
+      "learning_rate": 6.845333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2366
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.984375,
+      "epoch": 3.7872,
+      "grad_norm": 0.006691342741264224,
+      "kl": 0.096435546875,
+      "learning_rate": 6.844e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2367
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.078125,
+      "epoch": 3.7888,
+      "grad_norm": 9.683012814844659,
+      "kl": 0.097412109375,
+      "learning_rate": 6.842666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.828125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.828125,
+      "step": 2368
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.15625,
+      "epoch": 3.7904,
+      "grad_norm": 0.004789110616984344,
+      "kl": 0.090087890625,
+      "learning_rate": 6.841333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2369
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.375,
+      "epoch": 3.792,
+      "grad_norm": 0.005900148601139352,
+      "kl": 0.10107421875,
+      "learning_rate": 6.84e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 2370
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.421875,
+      "epoch": 3.7936,
+      "grad_norm": 8.352512971748757,
+      "kl": 0.095458984375,
+      "learning_rate": 6.838666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.59375,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.59375,
+      "step": 2371
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.15625,
+      "epoch": 3.7952,
+      "grad_norm": 0.010063234347460537,
+      "kl": 0.09912109375,
+      "learning_rate": 6.837333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2372
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.078125,
+      "epoch": 3.7968,
+      "grad_norm": 0.005138164255514939,
+      "kl": 0.0986328125,
+      "learning_rate": 6.836e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2373
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.9375,
+      "epoch": 3.7984,
+      "grad_norm": 0.018076274599385656,
+      "kl": 0.09814453125,
+      "learning_rate": 6.834666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 2374
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.03125,
+      "epoch": 3.8,
+      "grad_norm": 0.005843299540112735,
+      "kl": 0.104248046875,
+      "learning_rate": 6.833333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2375
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.015625,
+      "epoch": 3.8016,
+      "grad_norm": 0.005834186049690063,
+      "kl": 0.09716796875,
+      "learning_rate": 6.832e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2376
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.046875,
+      "epoch": 3.8032,
+      "grad_norm": 9.906937133651677,
+      "kl": 0.101806640625,
+      "learning_rate": 6.830666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.96875,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.96875,
+      "step": 2377
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.265625,
+      "epoch": 3.8048,
+      "grad_norm": 0.010573704984981475,
+      "kl": 0.100341796875,
+      "learning_rate": 6.829333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2378
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.453125,
+      "epoch": 3.8064,
+      "grad_norm": 0.00700351001140727,
+      "kl": 0.097900390625,
+      "learning_rate": 6.827999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2379
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.3125,
+      "epoch": 3.808,
+      "grad_norm": 8.177631549314306,
+      "kl": 0.102783203125,
+      "learning_rate": 6.826666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.765625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.765625,
+      "step": 2380
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.046875,
+      "epoch": 3.8096,
+      "grad_norm": 0.008134899117353633,
+      "kl": 0.0966796875,
+      "learning_rate": 6.825333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2381
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.65625,
+      "epoch": 3.8112,
+      "grad_norm": 0.023536452883869456,
+      "kl": 0.10498046875,
+      "learning_rate": 6.824e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2382
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.234375,
+      "epoch": 3.8128,
+      "grad_norm": 0.005522661498892688,
+      "kl": 0.098876953125,
+      "learning_rate": 6.822666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2383
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.765625,
+      "epoch": 3.8144,
+      "grad_norm": 0.006932339253504781,
+      "kl": 0.10107421875,
+      "learning_rate": 6.821333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2384
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.984375,
+      "epoch": 3.816,
+      "grad_norm": 0.00597749103319542,
+      "kl": 0.0947265625,
+      "learning_rate": 6.82e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2385
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.3125,
+      "epoch": 3.8176,
+      "grad_norm": 0.00547547211021991,
+      "kl": 0.09423828125,
+      "learning_rate": 6.818666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2386
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.078125,
+      "epoch": 3.8192,
+      "grad_norm": 0.0061512812840435315,
+      "kl": 0.0966796875,
+      "learning_rate": 6.817333333333333e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 2387
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 127.4375,
+      "epoch": 3.8208,
+      "grad_norm": 0.005619931377028051,
+      "kl": 0.093017578125,
+      "learning_rate": 6.816e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2388
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.53125,
+      "epoch": 3.8224,
+      "grad_norm": 0.00834837655748745,
+      "kl": 0.094970703125,
+      "learning_rate": 6.814666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2389
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.53125,
+      "epoch": 3.824,
+      "grad_norm": 0.006398271427540612,
+      "kl": 0.089599609375,
+      "learning_rate": 6.813333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2390
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.9375,
+      "epoch": 3.8256,
+      "grad_norm": 0.0063430160634467,
+      "kl": 0.100341796875,
+      "learning_rate": 6.812e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2391
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.375,
+      "epoch": 3.8272,
+      "grad_norm": 14.558043079604301,
+      "kl": 0.09130859375,
+      "learning_rate": 6.810666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.5625,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5625,
+      "step": 2392
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.734375,
+      "epoch": 3.8288,
+      "grad_norm": 0.00935651823755919,
+      "kl": 0.093994140625,
+      "learning_rate": 6.809333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2393
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.59375,
+      "epoch": 3.8304,
+      "grad_norm": 0.008013787494840057,
+      "kl": 0.095458984375,
+      "learning_rate": 6.807999999999999e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 2394
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.90625,
+      "epoch": 3.832,
+      "grad_norm": 0.008392254450809905,
+      "kl": 0.0927734375,
+      "learning_rate": 6.806666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2395
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.703125,
+      "epoch": 3.8336,
+      "grad_norm": 0.004741379046691836,
+      "kl": 0.09375,
+      "learning_rate": 6.805333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2396
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.015625,
+      "epoch": 3.8352,
+      "grad_norm": 0.006297647975464289,
+      "kl": 0.098388671875,
+      "learning_rate": 6.804e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2397
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.75,
+      "epoch": 3.8368,
+      "grad_norm": 0.006203005287109712,
+      "kl": 0.103271484375,
+      "learning_rate": 6.802666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 2398
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.796875,
+      "epoch": 3.8384,
+      "grad_norm": 53.8372722285475,
+      "kl": 0.101318359375,
+      "learning_rate": 6.801333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.546875,
+      "reward_std": 0.10205793008208275,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.546875,
+      "step": 2399
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.734375,
+      "epoch": 3.84,
+      "grad_norm": 0.006752821882955498,
+      "kl": 0.08837890625,
+      "learning_rate": 6.800000000000001e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 2400
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.625,
+      "epoch": 3.8416,
+      "grad_norm": 0.010986005482050473,
+      "kl": 0.099853515625,
+      "learning_rate": 6.798666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2401
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.453125,
+      "epoch": 3.8432,
+      "grad_norm": 117.43672215151724,
+      "kl": 0.099853515625,
+      "learning_rate": 6.797333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.359375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.359375,
+      "step": 2402
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.609375,
+      "epoch": 3.8448,
+      "grad_norm": 10.607061931115581,
+      "kl": 0.105224609375,
+      "learning_rate": 6.795999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 2403
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.546875,
+      "epoch": 3.8464,
+      "grad_norm": 0.004725759094412117,
+      "kl": 0.09326171875,
+      "learning_rate": 6.794666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2404
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.890625,
+      "epoch": 3.848,
+      "grad_norm": 0.0055892969207735925,
+      "kl": 0.09619140625,
+      "learning_rate": 6.793333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2405
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.359375,
+      "epoch": 3.8496,
+      "grad_norm": 31.36696531143331,
+      "kl": 0.107421875,
+      "learning_rate": 6.792e-07,
+      "loss": 0.0001,
+      "reward": 1.5625,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5625,
+      "step": 2406
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.03125,
+      "epoch": 3.8512,
+      "grad_norm": 0.0070162008730399366,
+      "kl": 0.11474609375,
+      "learning_rate": 6.790666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2407
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.625,
+      "epoch": 3.8528000000000002,
+      "grad_norm": 0.03487812032937742,
+      "kl": 0.100341796875,
+      "learning_rate": 6.789333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2408
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.046875,
+      "epoch": 3.8544,
+      "grad_norm": 0.009141754701625088,
+      "kl": 0.103759765625,
+      "learning_rate": 6.788e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2409
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.046875,
+      "epoch": 3.856,
+      "grad_norm": 0.005967400346223692,
+      "kl": 0.095458984375,
+      "learning_rate": 6.786666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2410
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.328125,
+      "epoch": 3.8576,
+      "grad_norm": 0.012273287291514253,
+      "kl": 0.105712890625,
+      "learning_rate": 6.785333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2411
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.65625,
+      "epoch": 3.8592,
+      "grad_norm": 0.010245456112344587,
+      "kl": 0.095458984375,
+      "learning_rate": 6.783999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2412
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.765625,
+      "epoch": 3.8608000000000002,
+      "grad_norm": 0.0047949738424549965,
+      "kl": 0.08642578125,
+      "learning_rate": 6.782666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2413
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.953125,
+      "epoch": 3.8624,
+      "grad_norm": 0.004382404452088223,
+      "kl": 0.09814453125,
+      "learning_rate": 6.781333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2414
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.796875,
+      "epoch": 3.864,
+      "grad_norm": 0.012478861380966813,
+      "kl": 0.09521484375,
+      "learning_rate": 6.78e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2415
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.828125,
+      "epoch": 3.8656,
+      "grad_norm": 0.011028077271733019,
+      "kl": 0.091552734375,
+      "learning_rate": 6.778666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2416
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.078125,
+      "epoch": 3.8672,
+      "grad_norm": 13.898095212594583,
+      "kl": 0.099609375,
+      "learning_rate": 6.777333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.953125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.953125,
+      "step": 2417
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.140625,
+      "epoch": 3.8688000000000002,
+      "grad_norm": 61.08251259004364,
+      "kl": 0.111328125,
+      "learning_rate": 6.776e-07,
+      "loss": 0.0001,
+      "reward": 1.84375,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.84375,
+      "step": 2418
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.578125,
+      "epoch": 3.8704,
+      "grad_norm": 0.05929140796658979,
+      "kl": 0.098388671875,
+      "learning_rate": 6.774666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2419
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 131.390625,
+      "epoch": 3.872,
+      "grad_norm": 0.0053238336217662505,
+      "kl": 0.092041015625,
+      "learning_rate": 6.773333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2420
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.234375,
+      "epoch": 3.8736,
+      "grad_norm": 0.007447370006031997,
+      "kl": 0.088623046875,
+      "learning_rate": 6.772e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2421
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.953125,
+      "epoch": 3.8752,
+      "grad_norm": 0.006028780972191757,
+      "kl": 0.095947265625,
+      "learning_rate": 6.770666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2422
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.3125,
+      "epoch": 3.8768000000000002,
+      "grad_norm": 29.47798801404022,
+      "kl": 0.08837890625,
+      "learning_rate": 6.769333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.953125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.953125,
+      "step": 2423
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.96875,
+      "epoch": 3.8784,
+      "grad_norm": 0.010966409825890451,
+      "kl": 0.0966796875,
+      "learning_rate": 6.767999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2424
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.578125,
+      "epoch": 3.88,
+      "grad_norm": 67.48194481594408,
+      "kl": 0.146728515625,
+      "learning_rate": 6.766666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.90625,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.90625,
+      "step": 2425
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.765625,
+      "epoch": 3.8816,
+      "grad_norm": 0.007271855308417963,
+      "kl": 0.099609375,
+      "learning_rate": 6.765333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2426
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.03125,
+      "epoch": 3.8832,
+      "grad_norm": 0.005827656443212807,
+      "kl": 0.08544921875,
+      "learning_rate": 6.764e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2427
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.1875,
+      "epoch": 3.8848000000000003,
+      "grad_norm": 0.004576782376676308,
+      "kl": 0.095458984375,
+      "learning_rate": 6.762666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.25,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.25,
+      "step": 2428
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.0625,
+      "epoch": 3.8864,
+      "grad_norm": 0.00638875497319546,
+      "kl": 0.08984375,
+      "learning_rate": 6.761333333333334e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 2429
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.40625,
+      "epoch": 3.888,
+      "grad_norm": 2.299140439688885,
+      "kl": 0.08984375,
+      "learning_rate": 6.76e-07,
+      "loss": 0.0001,
+      "reward": 1.640625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.640625,
+      "step": 2430
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.796875,
+      "epoch": 3.8895999999999997,
+      "grad_norm": 0.01237330702756149,
+      "kl": 0.0927734375,
+      "learning_rate": 6.758666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2431
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.84375,
+      "epoch": 3.8912,
+      "grad_norm": 0.004419196089592298,
+      "kl": 0.088134765625,
+      "learning_rate": 6.757333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2432
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 142.703125,
+      "epoch": 3.8928000000000003,
+      "grad_norm": 0.005460211163217887,
+      "kl": 0.089599609375,
+      "learning_rate": 6.755999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2433
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 143.890625,
+      "epoch": 3.8944,
+      "grad_norm": 0.009128321739199738,
+      "kl": 0.096923828125,
+      "learning_rate": 6.754666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2434
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.46875,
+      "epoch": 3.896,
+      "grad_norm": 0.005000159749408633,
+      "kl": 0.090087890625,
+      "learning_rate": 6.753333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 2435
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.234375,
+      "epoch": 3.8975999999999997,
+      "grad_norm": 0.00573971813711643,
+      "kl": 0.091552734375,
+      "learning_rate": 6.752e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2436
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 144.171875,
+      "epoch": 3.8992,
+      "grad_norm": 0.005423934479798023,
+      "kl": 0.092041015625,
+      "learning_rate": 6.750666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2437
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 147.875,
+      "epoch": 3.9008000000000003,
+      "grad_norm": 0.0048964918446726055,
+      "kl": 0.090087890625,
+      "learning_rate": 6.749333333333334e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 2438
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.8125,
+      "epoch": 3.9024,
+      "grad_norm": 0.005505343043529767,
+      "kl": 0.09130859375,
+      "learning_rate": 6.747999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2439
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 144.640625,
+      "epoch": 3.904,
+      "grad_norm": 0.006255989767989112,
+      "kl": 0.094970703125,
+      "learning_rate": 6.746666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2440
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 147.90625,
+      "epoch": 3.9055999999999997,
+      "grad_norm": 0.005957543880132684,
+      "kl": 0.088623046875,
+      "learning_rate": 6.745333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2441
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 142.859375,
+      "epoch": 3.9072,
+      "grad_norm": 14.617025169728223,
+      "kl": 0.08935546875,
+      "learning_rate": 6.744e-07,
+      "loss": 0.0001,
+      "reward": 1.765625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.765625,
+      "step": 2442
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 146.25,
+      "epoch": 3.9088000000000003,
+      "grad_norm": 0.006003524488151959,
+      "kl": 0.097412109375,
+      "learning_rate": 6.742666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2443
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 145.5625,
+      "epoch": 3.9104,
+      "grad_norm": 0.005808597031487233,
+      "kl": 0.088134765625,
+      "learning_rate": 6.741333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2444
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 146.265625,
+      "epoch": 3.912,
+      "grad_norm": 0.005917992806143221,
+      "kl": 0.09521484375,
+      "learning_rate": 6.74e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2445
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 147.875,
+      "epoch": 3.9135999999999997,
+      "grad_norm": 0.006172332394566476,
+      "kl": 0.0849609375,
+      "learning_rate": 6.738666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2446
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 146.609375,
+      "epoch": 3.9152,
+      "grad_norm": 0.0069262165477880396,
+      "kl": 0.092041015625,
+      "learning_rate": 6.737333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2447
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 151.1875,
+      "epoch": 3.9168,
+      "grad_norm": 0.005750141546430694,
+      "kl": 0.097900390625,
+      "learning_rate": 6.736e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2448
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 149.546875,
+      "epoch": 3.9184,
+      "grad_norm": 0.004324256501980477,
+      "kl": 0.088623046875,
+      "learning_rate": 6.734666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2449
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 144.296875,
+      "epoch": 3.92,
+      "grad_norm": 0.0045562474480998864,
+      "kl": 0.0810546875,
+      "learning_rate": 6.733333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.25,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.25,
+      "step": 2450
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 145.546875,
+      "epoch": 3.9215999999999998,
+      "grad_norm": 0.0047577648873752,
+      "kl": 0.088623046875,
+      "learning_rate": 6.732e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2451
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 155.390625,
+      "epoch": 3.9232,
+      "grad_norm": 0.053101827256318866,
+      "kl": 0.092529296875,
+      "learning_rate": 6.730666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2452
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 149.265625,
+      "epoch": 3.9248,
+      "grad_norm": 0.005270141562879986,
+      "kl": 0.093994140625,
+      "learning_rate": 6.729333333333334e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 2453
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 145.796875,
+      "epoch": 3.9264,
+      "grad_norm": 0.005040472267542715,
+      "kl": 0.093505859375,
+      "learning_rate": 6.727999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2454
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 151.28125,
+      "epoch": 3.928,
+      "grad_norm": 0.005384958125898764,
+      "kl": 0.08984375,
+      "learning_rate": 6.726666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2455
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 148.3125,
+      "epoch": 3.9295999999999998,
+      "grad_norm": 0.011129067414362587,
+      "kl": 0.095458984375,
+      "learning_rate": 6.725333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2456
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 152.140625,
+      "epoch": 3.9312,
+      "grad_norm": 0.00596080285866993,
+      "kl": 0.0849609375,
+      "learning_rate": 6.724e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2457
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 147.640625,
+      "epoch": 3.9328,
+      "grad_norm": 0.005333920087827407,
+      "kl": 0.089599609375,
+      "learning_rate": 6.722666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2458
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 147.0,
+      "epoch": 3.9344,
+      "grad_norm": 0.006154958987776238,
+      "kl": 0.087646484375,
+      "learning_rate": 6.721333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2459
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 154.765625,
+      "epoch": 3.936,
+      "grad_norm": 0.004479126161678747,
+      "kl": 0.084228515625,
+      "learning_rate": 6.72e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2460
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 153.0625,
+      "epoch": 3.9375999999999998,
+      "grad_norm": 40.56143423286713,
+      "kl": 0.090576171875,
+      "learning_rate": 6.718666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.859375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.859375,
+      "step": 2461
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 146.34375,
+      "epoch": 3.9392,
+      "grad_norm": 0.010070888387427798,
+      "kl": 0.091796875,
+      "learning_rate": 6.717333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2462
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 144.859375,
+      "epoch": 3.9408,
+      "grad_norm": 0.004684845434723455,
+      "kl": 0.08154296875,
+      "learning_rate": 6.716e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2463
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 147.109375,
+      "epoch": 3.9424,
+      "grad_norm": 0.0068647218138994165,
+      "kl": 0.08447265625,
+      "learning_rate": 6.714666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2464
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 143.71875,
+      "epoch": 3.944,
+      "grad_norm": 0.006093398884733159,
+      "kl": 0.095458984375,
+      "learning_rate": 6.713333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2465
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 149.6875,
+      "epoch": 3.9455999999999998,
+      "grad_norm": 0.004656264122214948,
+      "kl": 0.090087890625,
+      "learning_rate": 6.712e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 2466
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 155.40625,
+      "epoch": 3.9472,
+      "grad_norm": 0.0045984187700332906,
+      "kl": 0.080810546875,
+      "learning_rate": 6.710666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2467
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 152.953125,
+      "epoch": 3.9488,
+      "grad_norm": 0.007005373044540647,
+      "kl": 0.0849609375,
+      "learning_rate": 6.709333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2468
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 152.125,
+      "epoch": 3.9504,
+      "grad_norm": 0.007142292015770722,
+      "kl": 0.088134765625,
+      "learning_rate": 6.707999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2469
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 157.390625,
+      "epoch": 3.952,
+      "grad_norm": 0.006104431136933743,
+      "kl": 0.08740234375,
+      "learning_rate": 6.706666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2470
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 152.15625,
+      "epoch": 3.9536,
+      "grad_norm": 38.530285522947096,
+      "kl": 0.09423828125,
+      "learning_rate": 6.705333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.84375,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.84375,
+      "step": 2471
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 150.4375,
+      "epoch": 3.9552,
+      "grad_norm": 0.004828165857472615,
+      "kl": 0.0869140625,
+      "learning_rate": 6.704e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2472
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 150.328125,
+      "epoch": 3.9568,
+      "grad_norm": 0.006541928244353866,
+      "kl": 0.086669921875,
+      "learning_rate": 6.702666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2473
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 148.15625,
+      "epoch": 3.9584,
+      "grad_norm": 0.01481353657305946,
+      "kl": 0.0869140625,
+      "learning_rate": 6.701333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2474
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 153.78125,
+      "epoch": 3.96,
+      "grad_norm": 0.004475513670176121,
+      "kl": 0.080322265625,
+      "learning_rate": 6.7e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2475
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 143.90625,
+      "epoch": 3.9616,
+      "grad_norm": 0.005495556141375877,
+      "kl": 0.084716796875,
+      "learning_rate": 6.698666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2476
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 145.3125,
+      "epoch": 3.9632,
+      "grad_norm": 0.13879489931376993,
+      "kl": 0.085693359375,
+      "learning_rate": 6.697333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2477
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 149.265625,
+      "epoch": 3.9648,
+      "grad_norm": 10.426679231335926,
+      "kl": 0.08349609375,
+      "learning_rate": 6.695999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.6875,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.6875,
+      "step": 2478
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 153.0625,
+      "epoch": 3.9664,
+      "grad_norm": 0.005596128506336186,
+      "kl": 0.09130859375,
+      "learning_rate": 6.694666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2479
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 146.5625,
+      "epoch": 3.968,
+      "grad_norm": 0.005651955108786765,
+      "kl": 0.0888671875,
+      "learning_rate": 6.693333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2480
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 149.4375,
+      "epoch": 3.9696,
+      "grad_norm": 0.0053727431954373045,
+      "kl": 0.08642578125,
+      "learning_rate": 6.692e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2481
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 155.609375,
+      "epoch": 3.9712,
+      "grad_norm": 0.005185845111004155,
+      "kl": 0.0947265625,
+      "learning_rate": 6.690666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2482
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 143.953125,
+      "epoch": 3.9728,
+      "grad_norm": 0.007674500660243118,
+      "kl": 0.0830078125,
+      "learning_rate": 6.689333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2483
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 157.09375,
+      "epoch": 3.9744,
+      "grad_norm": 0.008117341236525024,
+      "kl": 0.08984375,
+      "learning_rate": 6.688e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2484
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 146.28125,
+      "epoch": 3.976,
+      "grad_norm": 0.0038480938157220653,
+      "kl": 0.0859375,
+      "learning_rate": 6.686666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2485
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 141.875,
+      "epoch": 3.9776,
+      "grad_norm": 0.005151737648553724,
+      "kl": 0.0810546875,
+      "learning_rate": 6.685333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2486
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 146.90625,
+      "epoch": 3.9792,
+      "grad_norm": 0.00999674541908965,
+      "kl": 0.09716796875,
+      "learning_rate": 6.683999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2487
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 148.6875,
+      "epoch": 3.9808,
+      "grad_norm": 0.006008420897788067,
+      "kl": 0.088623046875,
+      "learning_rate": 6.682666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2488
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 147.84375,
+      "epoch": 3.9824,
+      "grad_norm": 0.010053426298248316,
+      "kl": 0.091796875,
+      "learning_rate": 6.681333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2489
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 157.609375,
+      "epoch": 3.984,
+      "grad_norm": 0.0038157578789207113,
+      "kl": 0.084716796875,
+      "learning_rate": 6.68e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2490
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 151.65625,
+      "epoch": 3.9856,
+      "grad_norm": 0.011203376759661548,
+      "kl": 0.08447265625,
+      "learning_rate": 6.678666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2491
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 152.234375,
+      "epoch": 3.9872,
+      "grad_norm": 0.005372521773488301,
+      "kl": 0.07958984375,
+      "learning_rate": 6.677333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2492
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 146.390625,
+      "epoch": 3.9888,
+      "grad_norm": 0.004794573688539311,
+      "kl": 0.0859375,
+      "learning_rate": 6.676e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 2493
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 158.140625,
+      "epoch": 3.9904,
+      "grad_norm": 0.004067903832000818,
+      "kl": 0.081787109375,
+      "learning_rate": 6.674666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2494
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 148.984375,
+      "epoch": 3.992,
+      "grad_norm": 0.003940628123135005,
+      "kl": 0.082763671875,
+      "learning_rate": 6.673333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2495
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 148.171875,
+      "epoch": 3.9936,
+      "grad_norm": 21.50044649774401,
+      "kl": 0.081787109375,
+      "learning_rate": 6.671999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.40625,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.40625,
+      "step": 2496
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 147.796875,
+      "epoch": 3.9952,
+      "grad_norm": 0.0039060953941465414,
+      "kl": 0.0849609375,
+      "learning_rate": 6.670666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2497
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 146.15625,
+      "epoch": 3.9968,
+      "grad_norm": 0.004602306122436935,
+      "kl": 0.082275390625,
+      "learning_rate": 6.669333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 2498
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 151.890625,
+      "epoch": 3.9984,
+      "grad_norm": 0.007459849321689058,
+      "kl": 0.08935546875,
+      "learning_rate": 6.667999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2499
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 150.6875,
+      "epoch": 4.0,
+      "grad_norm": 0.004166730817293509,
+      "kl": 0.08056640625,
+      "learning_rate": 6.666666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2500
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 147.25,
+      "epoch": 4.0016,
+      "grad_norm": 0.009105099217947043,
+      "kl": 0.085205078125,
+      "learning_rate": 6.665333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2501
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 143.296875,
+      "epoch": 4.0032,
+      "grad_norm": 0.005567731071978663,
+      "kl": 0.08447265625,
+      "learning_rate": 6.664e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 2502
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 148.296875,
+      "epoch": 4.0048,
+      "grad_norm": 0.040379812397838714,
+      "kl": 0.093505859375,
+      "learning_rate": 6.662666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 2503
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 149.828125,
+      "epoch": 4.0064,
+      "grad_norm": 0.006876619346013394,
+      "kl": 0.087646484375,
+      "learning_rate": 6.661333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2504
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.8125,
+      "epoch": 4.008,
+      "grad_norm": 0.005219742283339008,
+      "kl": 0.089111328125,
+      "learning_rate": 6.66e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2505
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 155.3125,
+      "epoch": 4.0096,
+      "grad_norm": 38.6627755681772,
+      "kl": 0.092041015625,
+      "learning_rate": 6.658666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.640625,
+      "reward_std": 0.12255740165710449,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.640625,
+      "step": 2506
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 146.546875,
+      "epoch": 4.0112,
+      "grad_norm": 0.3057203368580988,
+      "kl": 0.08642578125,
+      "learning_rate": 6.657333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2507
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 150.8125,
+      "epoch": 4.0128,
+      "grad_norm": 32.19605404787532,
+      "kl": 0.08056640625,
+      "learning_rate": 6.655999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.6875,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.6875,
+      "step": 2508
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 154.0,
+      "epoch": 4.0144,
+      "grad_norm": 0.048335086906768666,
+      "kl": 0.08251953125,
+      "learning_rate": 6.654666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2509
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 158.515625,
+      "epoch": 4.016,
+      "grad_norm": 0.004003447413057817,
+      "kl": 0.085205078125,
+      "learning_rate": 6.653333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2510
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 150.203125,
+      "epoch": 4.0176,
+      "grad_norm": 0.009445237871011479,
+      "kl": 0.090087890625,
+      "learning_rate": 6.652e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2511
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 149.171875,
+      "epoch": 4.0192,
+      "grad_norm": 0.00950723920175294,
+      "kl": 0.08154296875,
+      "learning_rate": 6.650666666666667e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 2512
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 150.5,
+      "epoch": 4.0208,
+      "grad_norm": 0.0077344672441218235,
+      "kl": 0.095458984375,
+      "learning_rate": 6.649333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2513
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 147.390625,
+      "epoch": 4.0224,
+      "grad_norm": 0.006892087563494151,
+      "kl": 0.08203125,
+      "learning_rate": 6.647999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2514
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 157.171875,
+      "epoch": 4.024,
+      "grad_norm": 0.014666834877936281,
+      "kl": 0.08544921875,
+      "learning_rate": 6.646666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2515
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 148.3125,
+      "epoch": 4.0256,
+      "grad_norm": 0.004125663084657544,
+      "kl": 0.077880859375,
+      "learning_rate": 6.645333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2516
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 147.4375,
+      "epoch": 4.0272,
+      "grad_norm": 0.882004551617,
+      "kl": 0.08056640625,
+      "learning_rate": 6.643999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.71875,
+      "reward_std": 0.0883883461356163,
+      "rewards/format_reward": 0.984375,
+      "rewards/score_reward": 0.734375,
+      "step": 2517
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.0625,
+      "epoch": 4.0288,
+      "grad_norm": 29.568668128577638,
+      "kl": 0.0859375,
+      "learning_rate": 6.642666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.3125,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.3125,
+      "step": 2518
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.828125,
+      "epoch": 4.0304,
+      "grad_norm": 0.009374522478111123,
+      "kl": 0.079345703125,
+      "learning_rate": 6.641333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2519
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.015625,
+      "epoch": 4.032,
+      "grad_norm": 0.007946041981951719,
+      "kl": 0.08544921875,
+      "learning_rate": 6.64e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 2520
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.828125,
+      "epoch": 4.0336,
+      "grad_norm": 0.005442838576101794,
+      "kl": 0.084228515625,
+      "learning_rate": 6.638666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2521
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.796875,
+      "epoch": 4.0352,
+      "grad_norm": 0.00517555998000219,
+      "kl": 0.0830078125,
+      "learning_rate": 6.637333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2522
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.0,
+      "epoch": 4.0368,
+      "grad_norm": 0.0056392817455303305,
+      "kl": 0.087646484375,
+      "learning_rate": 6.636e-07,
+      "loss": 0.0001,
+      "reward": 1.25,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.25,
+      "step": 2523
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.734375,
+      "epoch": 4.0384,
+      "grad_norm": 0.0041004591927210465,
+      "kl": 0.08203125,
+      "learning_rate": 6.634666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2524
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.90625,
+      "epoch": 4.04,
+      "grad_norm": 0.004689123923517399,
+      "kl": 0.086181640625,
+      "learning_rate": 6.633333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2525
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.578125,
+      "epoch": 4.0416,
+      "grad_norm": 0.005929964781632597,
+      "kl": 0.09326171875,
+      "learning_rate": 6.632e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2526
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.84375,
+      "epoch": 4.0432,
+      "grad_norm": 8.828546663279477,
+      "kl": 0.094970703125,
+      "learning_rate": 6.630666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.484375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.484375,
+      "step": 2527
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.390625,
+      "epoch": 4.0448,
+      "grad_norm": 0.010442656220598324,
+      "kl": 0.099853515625,
+      "learning_rate": 6.629333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2528
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.03125,
+      "epoch": 4.0464,
+      "grad_norm": 0.004783679934250227,
+      "kl": 0.089111328125,
+      "learning_rate": 6.627999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2529
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.265625,
+      "epoch": 4.048,
+      "grad_norm": 0.011404404452718288,
+      "kl": 0.097900390625,
+      "learning_rate": 6.626666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2530
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.40625,
+      "epoch": 4.0496,
+      "grad_norm": 0.006376482448653157,
+      "kl": 0.08740234375,
+      "learning_rate": 6.625333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2531
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 117.640625,
+      "epoch": 4.0512,
+      "grad_norm": 0.005058889191245684,
+      "kl": 0.08984375,
+      "learning_rate": 6.624e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2532
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 118.921875,
+      "epoch": 4.0528,
+      "grad_norm": 0.005865295067476303,
+      "kl": 0.1015625,
+      "learning_rate": 6.622666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2533
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.0,
+      "epoch": 4.0544,
+      "grad_norm": 0.02178762532214028,
+      "kl": 0.09423828125,
+      "learning_rate": 6.621333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2534
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.6875,
+      "epoch": 4.056,
+      "grad_norm": 0.005286359410569147,
+      "kl": 0.094970703125,
+      "learning_rate": 6.62e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2535
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 114.90625,
+      "epoch": 4.0576,
+      "grad_norm": 0.006378572524974841,
+      "kl": 0.09326171875,
+      "learning_rate": 6.618666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2536
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.484375,
+      "epoch": 4.0592,
+      "grad_norm": 3.4905969355994393,
+      "kl": 0.096923828125,
+      "learning_rate": 6.617333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.65625,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.65625,
+      "step": 2537
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.40625,
+      "epoch": 4.0608,
+      "grad_norm": 0.005431200430068165,
+      "kl": 0.091552734375,
+      "learning_rate": 6.615999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2538
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.5625,
+      "epoch": 4.0624,
+      "grad_norm": 0.009549029141527297,
+      "kl": 0.099365234375,
+      "learning_rate": 6.614666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2539
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.25,
+      "epoch": 4.064,
+      "grad_norm": 0.004481870900867941,
+      "kl": 0.09375,
+      "learning_rate": 6.613333333333333e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 2540
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 126.71875,
+      "epoch": 4.0656,
+      "grad_norm": 0.005687143811485169,
+      "kl": 0.0927734375,
+      "learning_rate": 6.612e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2541
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.890625,
+      "epoch": 4.0672,
+      "grad_norm": 0.006445980965719126,
+      "kl": 0.085693359375,
+      "learning_rate": 6.610666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2542
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 120.96875,
+      "epoch": 4.0688,
+      "grad_norm": 0.012622437447995178,
+      "kl": 0.088134765625,
+      "learning_rate": 6.609333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2543
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 123.8125,
+      "epoch": 4.0704,
+      "grad_norm": 0.00490242038174239,
+      "kl": 0.0830078125,
+      "learning_rate": 6.608e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2544
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.46875,
+      "epoch": 4.072,
+      "grad_norm": 0.005267882298606897,
+      "kl": 0.087158203125,
+      "learning_rate": 6.606666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 2545
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.78125,
+      "epoch": 4.0736,
+      "grad_norm": 0.004240604299143198,
+      "kl": 0.08642578125,
+      "learning_rate": 6.605333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2546
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.21875,
+      "epoch": 4.0752,
+      "grad_norm": 0.006125720402937954,
+      "kl": 0.0927734375,
+      "learning_rate": 6.604e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2547
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.15625,
+      "epoch": 4.0768,
+      "grad_norm": 0.005125397988154121,
+      "kl": 0.08984375,
+      "learning_rate": 6.602666666666666e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 2548
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 121.796875,
+      "epoch": 4.0784,
+      "grad_norm": 0.004554422137870621,
+      "kl": 0.097900390625,
+      "learning_rate": 6.601333333333333e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 2549
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.375,
+      "epoch": 4.08,
+      "grad_norm": 10.879256191844307,
+      "kl": 0.083984375,
+      "learning_rate": 6.6e-07,
+      "loss": 0.0001,
+      "reward": 1.828125,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.828125,
+      "step": 2550
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.1875,
+      "epoch": 4.0816,
+      "grad_norm": 0.004857056004526661,
+      "kl": 0.0908203125,
+      "learning_rate": 6.598666666666667e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 2551
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.75,
+      "epoch": 4.0832,
+      "grad_norm": 38.60448821240214,
+      "kl": 0.08740234375,
+      "learning_rate": 6.597333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.71875,
+      "reward_std": 0.0578637570142746,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.71875,
+      "step": 2552
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 130.8125,
+      "epoch": 4.0848,
+      "grad_norm": 0.0055722457473076975,
+      "kl": 0.089111328125,
+      "learning_rate": 6.595999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2553
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 128.140625,
+      "epoch": 4.0864,
+      "grad_norm": 0.005901585658120285,
+      "kl": 0.0908203125,
+      "learning_rate": 6.594666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2554
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 125.21875,
+      "epoch": 4.088,
+      "grad_norm": 0.006043320919135986,
+      "kl": 0.1015625,
+      "learning_rate": 6.593333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2555
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.6875,
+      "epoch": 4.0896,
+      "grad_norm": 41.014813272034964,
+      "kl": 0.08203125,
+      "learning_rate": 6.592e-07,
+      "loss": 0.0001,
+      "reward": 1.734375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.734375,
+      "step": 2556
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 122.828125,
+      "epoch": 4.0912,
+      "grad_norm": 0.005623349192055146,
+      "kl": 0.091064453125,
+      "learning_rate": 6.590666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2557
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.734375,
+      "epoch": 4.0928,
+      "grad_norm": 0.004197994885303499,
+      "kl": 0.08447265625,
+      "learning_rate": 6.589333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2558
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.734375,
+      "epoch": 4.0944,
+      "grad_norm": 0.005377112077157976,
+      "kl": 0.09423828125,
+      "learning_rate": 6.588e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2559
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.96875,
+      "epoch": 4.096,
+      "grad_norm": 0.0048233082279107884,
+      "kl": 0.090087890625,
+      "learning_rate": 6.586666666666666e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 2560
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 124.390625,
+      "epoch": 4.0976,
+      "grad_norm": 0.0061077678231479,
+      "kl": 0.105712890625,
+      "learning_rate": 6.585333333333332e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 2561
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.25,
+      "epoch": 4.0992,
+      "grad_norm": 0.006672920623686077,
+      "kl": 0.086181640625,
+      "learning_rate": 6.583999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2562
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.15625,
+      "epoch": 4.1008,
+      "grad_norm": 0.006141821566205476,
+      "kl": 0.08984375,
+      "learning_rate": 6.582666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2563
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.234375,
+      "epoch": 4.1024,
+      "grad_norm": 0.008162071957256645,
+      "kl": 0.093505859375,
+      "learning_rate": 6.581333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.375,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.375,
+      "step": 2564
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.90625,
+      "epoch": 4.104,
+      "grad_norm": 0.004540971173253834,
+      "kl": 0.082275390625,
+      "learning_rate": 6.58e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2565
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.125,
+      "epoch": 4.1056,
+      "grad_norm": 0.03192894160850745,
+      "kl": 0.089111328125,
+      "learning_rate": 6.578666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2566
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.984375,
+      "epoch": 4.1072,
+      "grad_norm": 0.010552472332428767,
+      "kl": 0.07958984375,
+      "learning_rate": 6.577333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2567
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.875,
+      "epoch": 4.1088,
+      "grad_norm": 57.471915166964,
+      "kl": 0.096435546875,
+      "learning_rate": 6.576e-07,
+      "loss": 0.0001,
+      "reward": 1.640625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.640625,
+      "step": 2568
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.703125,
+      "epoch": 4.1104,
+      "grad_norm": 0.016453823309859207,
+      "kl": 0.080810546875,
+      "learning_rate": 6.574666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2569
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.109375,
+      "epoch": 4.112,
+      "grad_norm": 0.004277365551530939,
+      "kl": 0.08203125,
+      "learning_rate": 6.573333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2570
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.984375,
+      "epoch": 4.1136,
+      "grad_norm": 0.009120705745950298,
+      "kl": 0.089599609375,
+      "learning_rate": 6.571999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.5,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5,
+      "step": 2571
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.34375,
+      "epoch": 4.1152,
+      "grad_norm": 0.005383106684198995,
+      "kl": 0.083251953125,
+      "learning_rate": 6.570666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2572
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 141.546875,
+      "epoch": 4.1168,
+      "grad_norm": 12.588789098402401,
+      "kl": 0.093017578125,
+      "learning_rate": 6.569333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.859375,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.859375,
+      "step": 2573
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 136.734375,
+      "epoch": 4.1184,
+      "grad_norm": 0.017314804118025388,
+      "kl": 0.089111328125,
+      "learning_rate": 6.568e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2574
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.921875,
+      "epoch": 4.12,
+      "grad_norm": 0.011998078957182426,
+      "kl": 0.083740234375,
+      "learning_rate": 6.566666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2575
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.703125,
+      "epoch": 4.1216,
+      "grad_norm": 0.004444733298123726,
+      "kl": 0.08837890625,
+      "learning_rate": 6.565333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2576
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.484375,
+      "epoch": 4.1232,
+      "grad_norm": 0.007012174838558431,
+      "kl": 0.0771484375,
+      "learning_rate": 6.564e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2577
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.125,
+      "epoch": 4.1248,
+      "grad_norm": 0.004494631817878555,
+      "kl": 0.086181640625,
+      "learning_rate": 6.562666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2578
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 135.203125,
+      "epoch": 4.1264,
+      "grad_norm": 0.005123857321905895,
+      "kl": 0.09228515625,
+      "learning_rate": 6.561333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2579
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 133.46875,
+      "epoch": 4.128,
+      "grad_norm": 0.006726990656378754,
+      "kl": 0.083251953125,
+      "learning_rate": 6.56e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2580
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.234375,
+      "epoch": 4.1296,
+      "grad_norm": 0.008146626015911592,
+      "kl": 0.0849609375,
+      "learning_rate": 6.558666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2581
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.734375,
+      "epoch": 4.1312,
+      "grad_norm": 0.008148729184184638,
+      "kl": 0.086181640625,
+      "learning_rate": 6.557333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2582
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 142.203125,
+      "epoch": 4.1328,
+      "grad_norm": 0.011930567597164892,
+      "kl": 0.0927734375,
+      "learning_rate": 6.555999999999999e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2583
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.96875,
+      "epoch": 4.1344,
+      "grad_norm": 6.7273365040811415,
+      "kl": 0.083251953125,
+      "learning_rate": 6.554666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.640625,
+      "reward_std": 0.04419417306780815,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.640625,
+      "step": 2584
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 143.578125,
+      "epoch": 4.136,
+      "grad_norm": 0.0283230456033107,
+      "kl": 0.0830078125,
+      "learning_rate": 6.553333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2585
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 132.25,
+      "epoch": 4.1376,
+      "grad_norm": 0.004541300849407251,
+      "kl": 0.0830078125,
+      "learning_rate": 6.552e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2586
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 137.703125,
+      "epoch": 4.1392,
+      "grad_norm": 0.004323952954507166,
+      "kl": 0.07421875,
+      "learning_rate": 6.550666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2587
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.421875,
+      "epoch": 4.1408,
+      "grad_norm": 0.014045233973096672,
+      "kl": 0.078369140625,
+      "learning_rate": 6.549333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2588
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 129.609375,
+      "epoch": 4.1424,
+      "grad_norm": 0.006407366531407479,
+      "kl": 0.088623046875,
+      "learning_rate": 6.548000000000001e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2589
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.625,
+      "epoch": 4.144,
+      "grad_norm": 0.004146158822541215,
+      "kl": 0.083984375,
+      "learning_rate": 6.546666666666665e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2590
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.265625,
+      "epoch": 4.1456,
+      "grad_norm": 0.00843983808664062,
+      "kl": 0.0869140625,
+      "learning_rate": 6.545333333333332e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2591
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.21875,
+      "epoch": 4.1472,
+      "grad_norm": 0.010709284054382763,
+      "kl": 0.075927734375,
+      "learning_rate": 6.543999999999999e-07,
+      "loss": 0.0001,
+      "reward": 2.0,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 1.0,
+      "step": 2592
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 145.515625,
+      "epoch": 4.1488,
+      "grad_norm": 11.528906838149293,
+      "kl": 0.07568359375,
+      "learning_rate": 6.542666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.6875,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.6875,
+      "step": 2593
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 140.953125,
+      "epoch": 4.1504,
+      "grad_norm": 20.610375094519355,
+      "kl": 0.080810546875,
+      "learning_rate": 6.541333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.5625,
+      "reward_std": 0.06681530922651291,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.5625,
+      "step": 2594
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 144.9375,
+      "epoch": 4.152,
+      "grad_norm": 9.951367807093163,
+      "kl": 0.0771484375,
+      "learning_rate": 6.54e-07,
+      "loss": 0.0001,
+      "reward": 1.546875,
+      "reward_std": 0.0646936446428299,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.546875,
+      "step": 2595
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.75,
+      "epoch": 4.1536,
+      "grad_norm": 0.005442959414987329,
+      "kl": 0.09814453125,
+      "learning_rate": 6.538666666666667e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2596
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 134.78125,
+      "epoch": 4.1552,
+      "grad_norm": 0.013486451281767547,
+      "kl": 0.0830078125,
+      "learning_rate": 6.537333333333334e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2597
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 141.953125,
+      "epoch": 4.1568,
+      "grad_norm": 0.005227595505954613,
+      "kl": 0.07958984375,
+      "learning_rate": 6.536e-07,
+      "loss": 0.0001,
+      "reward": 1.875,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.875,
+      "step": 2598
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 138.078125,
+      "epoch": 4.1584,
+      "grad_norm": 0.012275691092826438,
+      "kl": 0.091552734375,
+      "learning_rate": 6.534666666666666e-07,
+      "loss": 0.0001,
+      "reward": 1.625,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.625,
+      "step": 2599
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 139.875,
+      "epoch": 4.16,
+      "grad_norm": 0.003973371009809525,
+      "kl": 0.071533203125,
+      "learning_rate": 6.533333333333333e-07,
+      "loss": 0.0001,
+      "reward": 1.75,
+      "reward_std": 0.0,
+      "rewards/format_reward": 1.0,
+      "rewards/score_reward": 0.75,
+      "step": 2600
+    }
+  ],
+  "logging_steps": 1.0,
+  "max_steps": 7500,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 12,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}