Model save

README.md

@@ -1,11 +1,9 @@
 ---
 base_model: Qwen/Qwen2.5-Math-7B
-datasets: DigitalLearningGmbH/MATH-lighteval
 library_name: transformers
 model_name: Qwen-2.5-7B-Simple-RL
 tags:
 - generated_from_trainer
-- open-r1
 - trl
 - grpo
 licence: license
@@ -13,7 +11,7 @@ licence: license
 
 # Model Card for Qwen-2.5-7B-Simple-RL
 
-This model is a fine-tuned version of [Qwen/Qwen2.5-Math-7B](https://huggingface.co/Qwen/Qwen2.5-Math-7B) on the [DigitalLearningGmbH/MATH-lighteval](https://huggingface.co/datasets/DigitalLearningGmbH/MATH-lighteval) dataset.
+This model is a fine-tuned version of [Qwen/Qwen2.5-Math-7B](https://huggingface.co/Qwen/Qwen2.5-Math-7B).
 It has been trained using [TRL](https://github.com/huggingface/trl).
 
 ## Quick start
@@ -29,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/jun-liang-chen-the-hong-kong-polytechnic-university/huggingface/runs/czqrqax3) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/jun-liang-chen-the-hong-kong-polytechnic-university/huggingface/runs/ye8fls8s) 
 
 
 This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).

all_results.json

@@ -1,13 +1,8 @@
 {
-    "eval_loss": 0.07094848901033401,
-    "eval_runtime": 47796.4045,
-    "eval_samples": 5000,
-    "eval_samples_per_second": 0.105,
-    "eval_steps_per_second": 0.017,
     "total_flos": 0.0,
-    "train_loss": 0.027036830155200475,
-    "train_runtime": 281082.0852,
+    "train_loss": 0.0008256121124141636,
+    "train_runtime": 419098.9196,
     "train_samples": 7500,
-    "train_samples_per_second": 0.027,
+    "train_samples_per_second": 0.018,
     "train_steps_per_second": 0.001
 }

config.json

@@ -23,7 +23,7 @@
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.49.0.dev0",
-  "use_cache": true,
+  "use_cache": false,
   "use_mrope": false,
   "use_sliding_window": false,
   "vocab_size": 152064

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:42712c9569b2980599252a40db264d2d1c08c116a7d552ec7b5bdb051d949b2b
+oid sha256:3d2460834ab32758c7929fd2127f4764dda55d37b2ed228b7d65398fdd13aa3f
 size 4877660776

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2e04adf4708f0377e2df413936055a264f94570ec1cf509b517212be76a1a036
+oid sha256:bcb46f0ac1dcead4a1ccca541f796c45a1d7d22a42f37e19cdcdf1b60bc0583c
 size 4932751008

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fe17ddeef939048814b042e4aa1e4f9ab07697fa5ab028c15f65110342623024
+oid sha256:9ec9437e98f69f030c1e09f702858d79bfacf2ed020e66df9be24b59c445c214
 size 4330865200

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f7dcdf332fdd5d2fb6991cc373e71db11897bebe7c65364593a931650149ca68
+oid sha256:fc740e9e6a6d8c353842a1ed8a64535fa9ce1baa0366313305dce8687dfe9175
 size 1089994880

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.027036830155200475,
-    "train_runtime": 281082.0852,
+    "train_loss": 0.0008256121124141636,
+    "train_runtime": 419098.9196,
     "train_samples": 7500,
-    "train_samples_per_second": 0.027,
+    "train_samples_per_second": 0.018,
     "train_steps_per_second": 0.001
 }

trainer_state.json

@@ -9,1074 +9,1074 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 427.0421132940995,
+      "completion_length": 644.0193140933388,
       "epoch": 0.012666666666666666,
-      "grad_norm": 0.32669389247894287,
-      "kl": 0.00012453716052205938,
+      "grad_norm": 0.16860076785087585,
+      "kl": 0.000124662016567431,
       "learning_rate": 3.75e-07,
       "loss": 0.0,
-      "reward": 0.41754386942637595,
-      "reward_std": 0.3627968850888704,
-      "rewards/accuracy_reward": 0.41754386942637595,
+      "reward": 0.605263171384209,
+      "reward_std": 0.40058013012534693,
+      "rewards/accuracy_reward": 0.605263171384209,
       "rewards/format_reward": 0.0,
       "step": 5
     },
     {
-      "completion_length": 425.203517552426,
+      "completion_length": 601.6614225688734,
       "epoch": 0.025333333333333333,
-      "grad_norm": 0.17229685187339783,
-      "kl": 0.00020355676349840667,
+      "grad_norm": 0.14201320707798004,
+      "kl": 0.00020345888639751233,
       "learning_rate": 7.5e-07,
       "loss": 0.0,
-      "reward": 0.44912281726535996,
-      "reward_std": 0.38365785103095207,
-      "rewards/accuracy_reward": 0.4473684310913086,
-      "rewards/format_reward": 0.0017543860171970569,
+      "reward": 0.673684226839166,
+      "reward_std": 0.39511305846666034,
+      "rewards/accuracy_reward": 0.673684226839166,
+      "rewards/format_reward": 0.0,
       "step": 10
     },
     {
-      "completion_length": 408.717552265368,
+      "completion_length": 579.6684371145149,
       "epoch": 0.038,
-      "grad_norm": 1.1077690124511719,
-      "kl": 0.000245641407213713,
+      "grad_norm": 0.3944288194179535,
+      "kl": 0.00023700312564247533,
       "learning_rate": 1.125e-06,
       "loss": 0.0,
-      "reward": 0.4894736951903293,
-      "reward_std": 0.3999988314352537,
-      "rewards/accuracy_reward": 0.4894736951903293,
+      "reward": 0.6631579123045269,
+      "reward_std": 0.3630171424464176,
+      "rewards/accuracy_reward": 0.6631579123045269,
       "rewards/format_reward": 0.0,
       "step": 15
     },
     {
-      "completion_length": 427.90176263106497,
+      "completion_length": 615.7789647152549,
       "epoch": 0.050666666666666665,
-      "grad_norm": 0.2539248466491699,
-      "kl": 0.0002711948595548931,
+      "grad_norm": 0.1742940992116928,
+      "kl": 0.0005895815397563733,
       "learning_rate": 1.5e-06,
       "loss": 0.0,
-      "reward": 0.41403509911737946,
-      "reward_std": 0.3837312685815912,
-      "rewards/accuracy_reward": 0.41403509911737946,
+      "reward": 0.6175438761711121,
+      "reward_std": 0.40319750308990476,
+      "rewards/accuracy_reward": 0.6175438761711121,
       "rewards/format_reward": 0.0,
       "step": 20
     },
     {
-      "completion_length": 433.0245695415296,
+      "completion_length": 653.6140499717311,
       "epoch": 0.06333333333333334,
-      "grad_norm": 0.2533121407032013,
-      "kl": 0.0006697002210115132,
+      "grad_norm": 0.213926300406456,
+      "kl": 0.0014986289174933182,
       "learning_rate": 1.875e-06,
       "loss": 0.0,
-      "reward": 0.45087720328255704,
-      "reward_std": 0.356808750880392,
-      "rewards/accuracy_reward": 0.45087720328255704,
+      "reward": 0.6561403632164001,
+      "reward_std": 0.3638197688679946,
+      "rewards/accuracy_reward": 0.6561403632164001,
       "rewards/format_reward": 0.0,
       "step": 25
     },
     {
-      "completion_length": 429.6824665270354,
+      "completion_length": 624.1438784950658,
       "epoch": 0.076,
-      "grad_norm": 0.5356761813163757,
-      "kl": 0.00229949951171875,
+      "grad_norm": 0.16140718758106232,
+      "kl": 0.0023109034488075658,
       "learning_rate": 2.25e-06,
-      "loss": 0.0001,
-      "reward": 0.4789473784597296,
-      "reward_std": 0.37421235348048965,
-      "rewards/accuracy_reward": 0.4789473784597296,
+      "loss": 0.0,
+      "reward": 0.6947368581044047,
+      "reward_std": 0.32959623650500647,
+      "rewards/accuracy_reward": 0.6947368581044047,
       "rewards/format_reward": 0.0,
       "step": 30
     },
     {
-      "completion_length": 457.07193699886926,
+      "completion_length": 649.8438781738281,
       "epoch": 0.08866666666666667,
-      "grad_norm": 0.1357738971710205,
-      "kl": 0.003329066226356908,
+      "grad_norm": 0.16571274399757385,
+      "kl": 0.009154390033922698,
       "learning_rate": 2.6250000000000003e-06,
-      "loss": 0.0001,
-      "reward": 0.4421052750788237,
-      "reward_std": 0.3569242841319034,
-      "rewards/accuracy_reward": 0.4421052750788237,
-      "rewards/format_reward": 0.0,
+      "loss": 0.0,
+      "reward": 0.6894737009939395,
+      "reward_std": 0.35860375699244046,
+      "rewards/accuracy_reward": 0.687719314819888,
+      "rewards/format_reward": 0.0017543860171970569,
       "step": 35
     },
     {
-      "completion_length": 445.33860441509046,
+      "completion_length": 629.4052797016344,
       "epoch": 0.10133333333333333,
-      "grad_norm": 0.10597766935825348,
-      "kl": 0.0031187559428967927,
+      "grad_norm": 0.27635857462882996,
+      "kl": 0.0027755938078227794,
       "learning_rate": 3e-06,
-      "loss": 0.0001,
-      "reward": 0.45789474844932554,
-      "reward_std": 0.4124853529428181,
-      "rewards/accuracy_reward": 0.45789474844932554,
+      "loss": 0.0,
+      "reward": 0.7035087861512836,
+      "reward_std": 0.34189380752412896,
+      "rewards/accuracy_reward": 0.7035087861512836,
       "rewards/format_reward": 0.0,
       "step": 40
     },
     {
-      "completion_length": 448.7719392475329,
+      "completion_length": 652.133349609375,
       "epoch": 0.114,
-      "grad_norm": 0.08034415543079376,
-      "kl": 0.003262630261872944,
+      "grad_norm": 0.0473485067486763,
+      "kl": 0.026468778911389802,
       "learning_rate": 2.998523534736735e-06,
-      "loss": 0.0001,
-      "reward": 0.46666667869216516,
-      "reward_std": 0.3817528630557813,
-      "rewards/accuracy_reward": 0.46666667869216516,
+      "loss": 0.0,
+      "reward": 0.6842105420012223,
+      "reward_std": 0.3472982666994396,
+      "rewards/accuracy_reward": 0.6842105420012223,
       "rewards/format_reward": 0.0,
       "step": 45
     },
     {
-      "completion_length": 443.4122892680921,
+      "completion_length": 628.0421226099918,
       "epoch": 0.12666666666666668,
-      "grad_norm": 0.14021478593349457,
-      "kl": 0.5420216209010074,
+      "grad_norm": 0.11586566269397736,
+      "kl": 0.004013121755499589,
       "learning_rate": 2.994097045546504e-06,
-      "loss": 0.0217,
-      "reward": 0.48421053682502946,
-      "reward_std": 0.32790782702596566,
-      "rewards/accuracy_reward": 0.48421053682502946,
+      "loss": 0.0,
+      "reward": 0.7263158050022627,
+      "reward_std": 0.3308417533573351,
+      "rewards/accuracy_reward": 0.7263158050022627,
       "rewards/format_reward": 0.0,
       "step": 50
     },
     {
-      "completion_length": 448.9508853310033,
+      "completion_length": 634.6701881810238,
       "epoch": 0.13933333333333334,
-      "grad_norm": 0.6121841073036194,
-      "kl": 0.006950779965049342,
+      "grad_norm": 0.09287750720977783,
+      "kl": 0.0033212159809313324,
       "learning_rate": 2.986729246506011e-06,
-      "loss": 0.0003,
-      "reward": 0.49122808010954605,
-      "reward_std": 0.3300354468195062,
-      "rewards/accuracy_reward": 0.49122808010954605,
+      "loss": 0.0,
+      "reward": 0.6912280863837192,
+      "reward_std": 0.35596638729697777,
+      "rewards/accuracy_reward": 0.6912280863837192,
       "rewards/format_reward": 0.0,
       "step": 55
     },
     {
-      "completion_length": 450.8982540732936,
+      "completion_length": 607.8982656378495,
       "epoch": 0.152,
-      "grad_norm": 0.1517709642648697,
-      "kl": 0.0043995104337993425,
+      "grad_norm": 0.1364029198884964,
+      "kl": 0.003622757761101974,
       "learning_rate": 2.976434642014389e-06,
-      "loss": 0.0002,
-      "reward": 0.4894736973862899,
-      "reward_std": 0.37877445064092935,
-      "rewards/accuracy_reward": 0.4894736973862899,
-      "rewards/format_reward": 0.0,
+      "loss": 0.0,
+      "reward": 0.7561403630595458,
+      "reward_std": 0.2851124694472865,
+      "rewards/accuracy_reward": 0.7543859771992031,
+      "rewards/format_reward": 0.0017543860171970569,
       "step": 60
     },
     {
-      "completion_length": 434.9368501362048,
+      "completion_length": 605.7438781738281,
       "epoch": 0.16466666666666666,
-      "grad_norm": 0.19329309463500977,
-      "kl": 0.003864890650699013,
+      "grad_norm": 0.0922616571187973,
+      "kl": 0.007114209626850329,
       "learning_rate": 2.9632334982395456e-06,
-      "loss": 0.0002,
-      "reward": 0.5245614123971839,
-      "reward_std": 0.35019483064350326,
-      "rewards/accuracy_reward": 0.5245614123971839,
+      "loss": 0.0,
+      "reward": 0.7578947509589948,
+      "reward_std": 0.29374928286201074,
+      "rewards/accuracy_reward": 0.7578947509589948,
       "rewards/format_reward": 0.0,
       "step": 65
     },
     {
-      "completion_length": 443.1368523848684,
+      "completion_length": 597.1175623843544,
       "epoch": 0.17733333333333334,
-      "grad_norm": 0.005799734033644199,
-      "kl": 0.0037913272255345395,
+      "grad_norm": 0.0,
+      "kl": 0.003088178132709704,
       "learning_rate": 2.947151803221774e-06,
-      "loss": 0.0002,
-      "reward": 0.4964912386317002,
-      "reward_std": 0.3756070852279663,
-      "rewards/accuracy_reward": 0.4964912386317002,
+      "loss": 0.0,
+      "reward": 0.7596491361919202,
+      "reward_std": 0.2632309951280293,
+      "rewards/accuracy_reward": 0.7596491361919202,
       "rewards/format_reward": 0.0,
       "step": 70
     },
     {
-      "completion_length": 451.34737677323193,
+      "completion_length": 599.9701933208265,
       "epoch": 0.19,
-      "grad_norm": 0.12151838839054108,
-      "kl": 0.004369795949835526,
+      "grad_norm": 0.10758549720048904,
+      "kl": 0.003361149838096217,
       "learning_rate": 2.928221215713164e-06,
-      "loss": 0.0002,
-      "reward": 0.470175449158016,
-      "reward_std": 0.38960387110710143,
-      "rewards/accuracy_reward": 0.470175449158016,
+      "loss": 0.0,
+      "reward": 0.7666666791627281,
+      "reward_std": 0.2686667611724452,
+      "rewards/accuracy_reward": 0.7666666791627281,
       "rewards/format_reward": 0.0,
       "step": 75
     },
     {
-      "completion_length": 434.12632606907897,
+      "completion_length": 597.1456305252879,
       "epoch": 0.20266666666666666,
-      "grad_norm": 0.2288283258676529,
-      "kl": 0.013469414961965461,
+      "grad_norm": 0.06053958460688591,
+      "kl": 0.0037446674547697368,
       "learning_rate": 2.906479002853542e-06,
-      "loss": 0.0005,
-      "reward": 0.48245615080783244,
-      "reward_std": 0.36099056883862146,
-      "rewards/accuracy_reward": 0.48245615080783244,
+      "loss": 0.0,
+      "reward": 0.7315789610147476,
+      "reward_std": 0.30245108447576824,
+      "rewards/accuracy_reward": 0.7315789610147476,
       "rewards/format_reward": 0.0,
       "step": 80
     },
     {
-      "completion_length": 437.27369481136924,
+      "completion_length": 577.8438778525905,
       "epoch": 0.21533333333333332,
-      "grad_norm": 0.18831907212734222,
-      "kl": 0.015026534231085527,
+      "grad_norm": 0.0840262621641159,
+      "kl": 0.004712596692537006,
       "learning_rate": 2.8819679668056195e-06,
-      "loss": 0.0006,
-      "reward": 0.5157894862325568,
-      "reward_std": 0.39665700228590717,
-      "rewards/accuracy_reward": 0.5157894862325568,
+      "loss": 0.0,
+      "reward": 0.7982456276291295,
+      "reward_std": 0.26917701614530465,
+      "rewards/accuracy_reward": 0.7982456276291295,
       "rewards/format_reward": 0.0,
       "step": 85
     },
     {
-      "completion_length": 440.3596586528577,
+      "completion_length": 612.2982611405222,
       "epoch": 0.228,
-      "grad_norm": 0.004888650495558977,
-      "kl": 2.206267828690378,
+      "grad_norm": 0.0,
+      "kl": 0.02382647865696957,
       "learning_rate": 2.8547363604937856e-06,
-      "loss": 0.0884,
-      "reward": 0.49298246753843206,
-      "reward_std": 0.3442464342242793,
-      "rewards/accuracy_reward": 0.49298246753843206,
+      "loss": 0.0,
+      "reward": 0.7456140491523241,
+      "reward_std": 0.2956205848016237,
+      "rewards/accuracy_reward": 0.7456140491523241,
       "rewards/format_reward": 0.0,
       "step": 90
     },
     {
-      "completion_length": 442.4719376413446,
+      "completion_length": 598.1403680098684,
       "epoch": 0.24066666666666667,
-      "grad_norm": 0.1227022334933281,
-      "kl": 0.0068352950246710524,
+      "grad_norm": 0.0,
+      "kl": 0.005134462055407072,
       "learning_rate": 2.824837792612416e-06,
-      "loss": 0.0003,
-      "reward": 0.5210526447547109,
-      "reward_std": 0.33440621652101216,
-      "rewards/accuracy_reward": 0.5210526447547109,
+      "loss": 0.0,
+      "reward": 0.7403508889047723,
+      "reward_std": 0.2658375608293634,
+      "rewards/accuracy_reward": 0.7403508889047723,
       "rewards/format_reward": 0.0,
       "step": 95
     },
     {
-      "completion_length": 431.6807120875308,
+      "completion_length": 582.6561587685032,
       "epoch": 0.25333333333333335,
-      "grad_norm": 0.18928714096546173,
-      "kl": 0.008197182103207237,
+      "grad_norm": 0.11761737614870071,
+      "kl": 0.00578155517578125,
       "learning_rate": 2.792331122090709e-06,
-      "loss": 0.0003,
-      "reward": 0.5385965028875752,
-      "reward_std": 0.35269192080748707,
-      "rewards/accuracy_reward": 0.5385965028875752,
+      "loss": 0.0,
+      "reward": 0.7614035217385543,
+      "reward_std": 0.25812496166480214,
+      "rewards/accuracy_reward": 0.7614035217385543,
       "rewards/format_reward": 0.0,
       "step": 100
     },
     {
       "epoch": 0.25333333333333335,
-      "eval_completion_length": 435.9969423065186,
-      "eval_kl": 0.009833270263671876,
-      "eval_loss": 0.00039347587153315544,
-      "eval_reward": 0.4568000102430582,
-      "eval_reward_std": 0.3654944499373436,
-      "eval_rewards/accuracy_reward": 0.4568000102430582,
+      "eval_completion_length": 576.5513494110107,
+      "eval_kl": 0.10147758865356446,
+      "eval_loss": 3.062895004291022e-08,
+      "eval_reward": 0.6709333465278149,
+      "eval_reward_std": 0.3338339452087879,
+      "eval_rewards/accuracy_reward": 0.6709333465278149,
       "eval_rewards/format_reward": 0.0,
-      "eval_runtime": 48530.4928,
-      "eval_samples_per_second": 0.103,
-      "eval_steps_per_second": 0.017,
+      "eval_runtime": 79047.156,
+      "eval_samples_per_second": 0.063,
+      "eval_steps_per_second": 0.011,
       "step": 100
     },
     {
-      "completion_length": 437.9824650814659,
+      "completion_length": 549.310542056435,
       "epoch": 0.266,
-      "grad_norm": 0.12078626453876495,
-      "kl": 0.008320858604029605,
+      "grad_norm": 0.08108708262443542,
+      "kl": 0.07660410027754934,
       "learning_rate": 2.7572803422217976e-06,
-      "loss": 0.0003,
-      "reward": 0.5263158009240502,
-      "reward_std": 0.37190421505978233,
-      "rewards/accuracy_reward": 0.5263158009240502,
+      "loss": 0.0,
+      "reward": 0.8140351000585054,
+      "reward_std": 0.2548168144728008,
+      "rewards/accuracy_reward": 0.8140351000585054,
       "rewards/format_reward": 0.0,
       "step": 105
     },
     {
-      "completion_length": 439.51228991056746,
+      "completion_length": 596.6351042094983,
       "epoch": 0.2786666666666667,
-      "grad_norm": 0.13941198587417603,
-      "kl": 0.0077880859375,
+      "grad_norm": 0.06687445938587189,
+      "kl": 0.18131464907997533,
       "learning_rate": 2.71975445468425e-06,
-      "loss": 0.0003,
-      "reward": 0.49122808167808935,
-      "reward_std": 0.4313258014227215,
-      "rewards/accuracy_reward": 0.49122808167808935,
+      "loss": 0.0,
+      "reward": 0.7543859803362897,
+      "reward_std": 0.32687413692474365,
+      "rewards/accuracy_reward": 0.7543859803362897,
       "rewards/format_reward": 0.0,
       "step": 110
     },
     {
-      "completion_length": 421.4017642372533,
+      "completion_length": 547.8263289602179,
       "epoch": 0.29133333333333333,
-      "grad_norm": 0.10381676256656647,
-      "kl": 0.00989588687294408,
+      "grad_norm": 0.08061777800321579,
+      "kl": 4.457168900339227,
       "learning_rate": 2.679827333703964e-06,
-      "loss": 0.0004,
-      "reward": 0.5719298359594847,
-      "reward_std": 0.3785097053176478,
-      "rewards/accuracy_reward": 0.5719298359594847,
+      "loss": -0.0,
+      "reward": 0.7894736992685418,
+      "reward_std": 0.2753143636803878,
+      "rewards/accuracy_reward": 0.7894736992685418,
       "rewards/format_reward": 0.0,
       "step": 115
     },
     {
-      "completion_length": 421.6526407342208,
+      "completion_length": 533.67895989669,
       "epoch": 0.304,
-      "grad_norm": 0.2092106193304062,
-      "kl": 0.010035464638157895,
+      "grad_norm": 0.15190833806991577,
+      "kl": 0.005019980982730263,
       "learning_rate": 2.637577580623858e-06,
-      "loss": 0.0004,
-      "reward": 0.5859649266067304,
-      "reward_std": 0.3874762525683955,
-      "rewards/accuracy_reward": 0.5859649266067304,
+      "loss": 0.0,
+      "reward": 0.771929837528028,
+      "reward_std": 0.2711241148020092,
+      "rewards/accuracy_reward": 0.771929837528028,
       "rewards/format_reward": 0.0,
       "step": 120
     },
     {
-      "completion_length": 414.329833984375,
+      "completion_length": 535.5947535464638,
       "epoch": 0.31666666666666665,
-      "grad_norm": 0.23555059731006622,
-      "kl": 0.009284732216282895,
+      "grad_norm": 0.24402964115142822,
+      "kl": 0.005212723581414474,
       "learning_rate": 2.593088369167671e-06,
-      "loss": 0.0004,
-      "reward": 0.62280702794853,
-      "reward_std": 0.32933386564254763,
-      "rewards/accuracy_reward": 0.62280702794853,
+      "loss": 0.0,
+      "reward": 0.8035087809750908,
+      "reward_std": 0.2135340957265151,
+      "rewards/accuracy_reward": 0.8035087809750908,
       "rewards/format_reward": 0.0,
       "step": 125
     },
     {
-      "completion_length": 417.16316656815377,
+      "completion_length": 532.9789643940172,
       "epoch": 0.3293333333333333,
-      "grad_norm": 0.12066253274679184,
-      "kl": 0.012572599712171053,
+      "grad_norm": 0.28009745478630066,
+      "kl": 0.005336239463404605,
       "learning_rate": 2.5464472817024772e-06,
-      "loss": 0.0005,
-      "reward": 0.6175438719360452,
-      "reward_std": 0.3299596526120838,
-      "rewards/accuracy_reward": 0.6175438719360452,
+      "loss": 0.0,
+      "reward": 0.8052631699725201,
+      "reward_std": 0.24214739736757782,
+      "rewards/accuracy_reward": 0.8052631699725201,
       "rewards/format_reward": 0.0,
       "step": 130
     },
     {
-      "completion_length": 429.32106323242186,
+      "completion_length": 561.8772093120374,
       "epoch": 0.342,
-      "grad_norm": 0.14316421747207642,
-      "kl": 0.015025570518092106,
+      "grad_norm": 0.18070097267627716,
+      "kl": 0.004754076505962171,
       "learning_rate": 2.497746136822254e-06,
-      "loss": 0.0006,
-      "reward": 0.5842105401189703,
-      "reward_std": 0.3870730996131897,
-      "rewards/accuracy_reward": 0.5842105401189703,
+      "loss": 0.0,
+      "reward": 0.7771929958933278,
+      "reward_std": 0.27938908150321556,
+      "rewards/accuracy_reward": 0.7771929958933278,
       "rewards/format_reward": 0.0,
       "step": 135
     },
     {
-      "completion_length": 429.531587942023,
+      "completion_length": 578.6684380782278,
       "epoch": 0.3546666666666667,
-      "grad_norm": 0.14851997792720795,
-      "kl": 0.02211560701069079,
+      "grad_norm": 0.10906434804201126,
+      "kl": 0.005059493215460526,
       "learning_rate": 2.4470808085919304e-06,
-      "loss": 0.0009,
-      "reward": 0.5649122933023855,
-      "reward_std": 0.35544770171767787,
-      "rewards/accuracy_reward": 0.5649122933023855,
+      "loss": 0.0,
+      "reward": 0.7631579088537317,
+      "reward_std": 0.3001742513556229,
+      "rewards/accuracy_reward": 0.7631579088537317,
       "rewards/format_reward": 0.0,
       "step": 140
     },
     {
-      "completion_length": 447.08246427837173,
+      "completion_length": 602.6772094726563,
       "epoch": 0.36733333333333335,
-      "grad_norm": 0.10792040824890137,
-      "kl": 0.017962325246710525,
+      "grad_norm": 0.05584894120693207,
+      "kl": 0.004565590306332237,
       "learning_rate": 2.3945510378077523e-06,
-      "loss": 0.0007,
-      "reward": 0.5122807138844541,
-      "reward_std": 0.35567033479088234,
-      "rewards/accuracy_reward": 0.5122807138844541,
+      "loss": 0.0,
+      "reward": 0.7842105371387381,
+      "reward_std": 0.2481692125922755,
+      "rewards/accuracy_reward": 0.7842105371387381,
       "rewards/format_reward": 0.0,
       "step": 145
     },
     {
-      "completion_length": 423.1122905530428,
+      "completion_length": 580.9210699784128,
       "epoch": 0.38,
-      "grad_norm": 0.11955911666154861,
-      "kl": 0.02356021278782895,
+      "grad_norm": 0.0609310120344162,
+      "kl": 0.005594434236225329,
       "learning_rate": 2.340260235645519e-06,
-      "loss": 0.0009,
-      "reward": 0.5578947500178688,
-      "reward_std": 0.36022400009004696,
-      "rewards/accuracy_reward": 0.5578947500178688,
+      "loss": 0.0,
+      "reward": 0.7368421157723979,
+      "reward_std": 0.26106126277070296,
+      "rewards/accuracy_reward": 0.7368421157723979,
       "rewards/format_reward": 0.0,
       "step": 150
     },
     {
-      "completion_length": 409.61053514982524,
+      "completion_length": 525.4000133313631,
       "epoch": 0.39266666666666666,
-      "grad_norm": 0.09739726036787033,
-      "kl": 0.018040385999177632,
+      "grad_norm": 0.06953240185976028,
+      "kl": 0.007059599223889802,
       "learning_rate": 2.2843152800832416e-06,
-      "loss": 0.0007,
-      "reward": 0.6157894856051395,
-      "reward_std": 0.34050145086489225,
-      "rewards/accuracy_reward": 0.6157894856051395,
+      "loss": 0.0,
+      "reward": 0.8035087816025082,
+      "reward_std": 0.2169914556177039,
+      "rewards/accuracy_reward": 0.8035087816025082,
       "rewards/format_reward": 0.0,
       "step": 155
     },
     {
-      "completion_length": 404.34913137335525,
+      "completion_length": 559.4386140522204,
       "epoch": 0.4053333333333333,
-      "grad_norm": 0.004406505264341831,
-      "kl": 0.02299708316200658,
+      "grad_norm": 0.08336005359888077,
+      "kl": 0.007145851536800986,
       "learning_rate": 2.2268263054989753e-06,
-      "loss": 0.0009,
-      "reward": 0.5842105368250294,
-      "reward_std": 0.3406506648189143,
-      "rewards/accuracy_reward": 0.5842105368250294,
+      "loss": 0.0,
+      "reward": 0.7824561511215411,
+      "reward_std": 0.23924477821902224,
+      "rewards/accuracy_reward": 0.7824561511215411,
       "rewards/format_reward": 0.0,
       "step": 160
     },
     {
-      "completion_length": 407.5386057000411,
+      "completion_length": 546.5772112394634,
       "epoch": 0.418,
-      "grad_norm": 0.18651004135608673,
-      "kl": 0.05064440275493421,
+      "grad_norm": 0.0791773647069931,
+      "kl": 0.0061695299650493425,
       "learning_rate": 2.167906485858047e-06,
-      "loss": 0.002,
-      "reward": 0.6210526471075258,
-      "reward_std": 0.32974544889048524,
-      "rewards/accuracy_reward": 0.6210526471075258,
+      "loss": 0.0,
+      "reward": 0.803508786308138,
+      "reward_std": 0.25981574560466564,
+      "rewards/accuracy_reward": 0.803508786308138,
       "rewards/format_reward": 0.0,
       "step": 165
     },
     {
-      "completion_length": 409.0772017629523,
+      "completion_length": 568.9280845240543,
       "epoch": 0.43066666666666664,
-      "grad_norm": 0.13046123087406158,
-      "kl": 0.06625655324835526,
+      "grad_norm": 0.11959535628557205,
+      "kl": 0.007060161389802632,
       "learning_rate": 2.1076718119164804e-06,
-      "loss": 0.0026,
-      "reward": 0.5631579061872081,
-      "reward_std": 0.37127605362942345,
-      "rewards/accuracy_reward": 0.5631579061872081,
+      "loss": 0.0,
+      "reward": 0.7736842216629731,
+      "reward_std": 0.2649867666395087,
+      "rewards/accuracy_reward": 0.7736842216629731,
       "rewards/format_reward": 0.0,
       "step": 170
     },
     {
-      "completion_length": 402.12983334189965,
+      "completion_length": 550.3982609799034,
       "epoch": 0.44333333333333336,
-      "grad_norm": 0.10294575989246368,
-      "kl": 0.027840383429276316,
+      "grad_norm": 0.0,
+      "kl": 0.007817157946134868,
       "learning_rate": 2.0462408628792335e-06,
-      "loss": 0.0011,
-      "reward": 0.6105263289652373,
-      "reward_std": 0.35427797938648026,
-      "rewards/accuracy_reward": 0.6105263289652373,
+      "loss": 0.0,
+      "reward": 0.7824561511215411,
+      "reward_std": 0.253275244800668,
+      "rewards/accuracy_reward": 0.7824561511215411,
       "rewards/format_reward": 0.0,
       "step": 175
     },
     {
-      "completion_length": 419.62632414165296,
+      "completion_length": 586.7596660413241,
       "epoch": 0.456,
-      "grad_norm": 0.20833256840705872,
-      "kl": 0.029789974814967105,
+      "grad_norm": 0.06831427663564682,
+      "kl": 0.0069172106291118425,
       "learning_rate": 1.9837345729627633e-06,
-      "loss": 0.0012,
-      "reward": 0.5614035218954087,
-      "reward_std": 0.3542021836105146,
-      "rewards/accuracy_reward": 0.5614035218954087,
+      "loss": 0.0,
+      "reward": 0.7438596612528751,
+      "reward_std": 0.2811869652647721,
+      "rewards/accuracy_reward": 0.7438596612528751,
       "rewards/format_reward": 0.0,
       "step": 180
     },
     {
-      "completion_length": 417.81930590177836,
+      "completion_length": 583.1228234542043,
       "epoch": 0.4686666666666667,
-      "grad_norm": 0.012007645331323147,
-      "kl": 0.02894447728207237,
+      "grad_norm": 0.10550453513860703,
+      "kl": 0.006921226099917763,
       "learning_rate": 1.9202759933214665e-06,
-      "loss": 0.0012,
-      "reward": 0.5105263262987136,
-      "reward_std": 0.34277828335762023,
-      "rewards/accuracy_reward": 0.5105263262987136,
+      "loss": 0.0,
+      "reward": 0.7631579093242946,
+      "reward_std": 0.29657848289138394,
+      "rewards/accuracy_reward": 0.7631579093242946,
       "rewards/format_reward": 0.0,
       "step": 185
     },
     {
-      "completion_length": 395.12457034462375,
+      "completion_length": 528.0912424187911,
       "epoch": 0.48133333333333334,
-      "grad_norm": 0.15910613536834717,
-      "kl": 0.031014532791940788,
+      "grad_norm": 0.0677071213722229,
+      "kl": 0.009208438270970395,
       "learning_rate": 1.8559900498066726e-06,
-      "loss": 0.0012,
-      "reward": 0.5754386062684812,
-      "reward_std": 0.31805680268689207,
-      "rewards/accuracy_reward": 0.5754386062684812,
+      "loss": 0.0,
+      "reward": 0.8140350991173795,
+      "reward_std": 0.22778719381282203,
+      "rewards/accuracy_reward": 0.8140350991173795,
       "rewards/format_reward": 0.0,
       "step": 190
     },
     {
-      "completion_length": 399.89123856393917,
+      "completion_length": 550.7807165848582,
       "epoch": 0.494,
-      "grad_norm": 0.20715074241161346,
-      "kl": 0.03553081311677632,
+      "grad_norm": 0.18718236684799194,
+      "kl": 0.008406307822779605,
       "learning_rate": 1.7910032970350677e-06,
-      "loss": 0.0014,
-      "reward": 0.5456140458583831,
-      "reward_std": 0.37994654994261895,
-      "rewards/accuracy_reward": 0.5456140458583831,
+      "loss": 0.0,
+      "reward": 0.7789473815968162,
+      "reward_std": 0.2719327967417868,
+      "rewards/accuracy_reward": 0.7789473815968162,
       "rewards/format_reward": 0.0,
       "step": 195
     },
     {
-      "completion_length": 407.15264089483964,
+      "completion_length": 551.1315929212068,
       "epoch": 0.5066666666666667,
-      "grad_norm": 0.29204627871513367,
-      "kl": 0.03386648077713816,
+      "grad_norm": 0.08260176330804825,
+      "kl": 0.00812426115337171,
       "learning_rate": 1.7254436692507058e-06,
-      "loss": 0.0014,
-      "reward": 0.5596491352507943,
-      "reward_std": 0.3601168986998106,
-      "rewards/accuracy_reward": 0.5596491352507943,
+      "loss": 0.0,
+      "reward": 0.8070175547348826,
+      "reward_std": 0.23249007557567797,
+      "rewards/accuracy_reward": 0.8070175547348826,
       "rewards/format_reward": 0.0,
       "step": 200
     },
     {
       "epoch": 0.5066666666666667,
-      "eval_completion_length": 394.2290429840088,
-      "eval_kl": 0.04040946655273438,
-      "eval_loss": 0.0015893690288066864,
-      "eval_reward": 0.49966667771935463,
-      "eval_reward_std": 0.3627293815135956,
-      "eval_rewards/accuracy_reward": 0.49966667771935463,
+      "eval_completion_length": 553.8582824371338,
+      "eval_kl": 0.02361789245605469,
+      "eval_loss": -4.30000654887408e-05,
+      "eval_reward": 0.6822333461046219,
+      "eval_reward_std": 0.32350232841968535,
+      "eval_rewards/accuracy_reward": 0.6822333461046219,
       "eval_rewards/format_reward": 0.0,
-      "eval_runtime": 47257.0811,
-      "eval_samples_per_second": 0.106,
-      "eval_steps_per_second": 0.018,
+      "eval_runtime": 76484.7991,
+      "eval_samples_per_second": 0.065,
+      "eval_steps_per_second": 0.011,
       "step": 200
     },
     {
-      "completion_length": 397.5421145790502,
+      "completion_length": 570.7842259457237,
       "epoch": 0.5193333333333333,
-      "grad_norm": 0.1434248983860016,
-      "kl": 0.046006373355263155,
+      "grad_norm": 0.14592121541500092,
+      "kl": 0.024847893965871712,
       "learning_rate": 1.6594402284710481e-06,
-      "loss": 0.0018,
-      "reward": 0.6000000109798029,
-      "reward_std": 0.3170362949371338,
-      "rewards/accuracy_reward": 0.6000000109798029,
+      "loss": 0.0,
+      "reward": 0.7561403608635853,
+      "reward_std": 0.24963736345893459,
+      "rewards/accuracy_reward": 0.7561403608635853,
       "rewards/format_reward": 0.0,
       "step": 205
     },
     {
-      "completion_length": 389.34562474300986,
+      "completion_length": 556.5035239771793,
       "epoch": 0.532,
-      "grad_norm": 0.21728208661079407,
-      "kl": 0.06839278371710526,
+      "grad_norm": 0.06877300888299942,
+      "kl": 0.011327321905838815,
       "learning_rate": 1.593122910412851e-06,
-      "loss": 0.0027,
-      "reward": 0.591228081991798,
-      "reward_std": 0.3792112864946064,
-      "rewards/accuracy_reward": 0.591228081991798,
+      "loss": 0.0,
+      "reward": 0.8210526436567307,
+      "reward_std": 0.22859587669372558,
+      "rewards/accuracy_reward": 0.8210526436567307,
       "rewards/format_reward": 0.0,
       "step": 210
     },
     {
-      "completion_length": 393.13334222090873,
+      "completion_length": 579.1263334575452,
       "epoch": 0.5446666666666666,
-      "grad_norm": 4.170770168304443,
-      "kl": 0.08757773951480263,
+      "grad_norm": 0.0,
+      "kl": 0.007683683696546053,
       "learning_rate": 1.5266222686980693e-06,
-      "loss": 0.0035,
-      "reward": 0.41929825356132105,
-      "reward_std": 0.33843882052521956,
-      "rewards/accuracy_reward": 0.41929825356132105,
+      "loss": 0.0,
+      "reward": 0.7561403643143805,
+      "reward_std": 0.2832074833543677,
+      "rewards/accuracy_reward": 0.7561403643143805,
       "rewards/format_reward": 0.0,
       "step": 215
     },
     {
-      "completion_length": 402.3175527472245,
+      "completion_length": 588.4649293598376,
       "epoch": 0.5573333333333333,
-      "grad_norm": 0.04682053253054619,
-      "kl": 0.10470291940789474,
+      "grad_norm": 0.07735468447208405,
+      "kl": 0.007682238127055921,
       "learning_rate": 1.460069217843338e-06,
-      "loss": 0.0042,
-      "reward": 0.3122807102768045,
-      "reward_std": 0.35611560125099984,
-      "rewards/accuracy_reward": 0.3122807102768045,
+      "loss": 0.0,
+      "reward": 0.7333333465613817,
+      "reward_std": 0.31152473405787817,
+      "rewards/accuracy_reward": 0.7333333465613817,
       "rewards/format_reward": 0.0,
       "step": 220
     },
     {
-      "completion_length": 421.3649226138466,
+      "completion_length": 582.8508938438014,
       "epoch": 0.57,
-      "grad_norm": 0.3474079668521881,
-      "kl": 0.09076377467105264,
+      "grad_norm": 0.04747384414076805,
+      "kl": 0.008666349712171053,
       "learning_rate": 1.3935947755389924e-06,
-      "loss": 0.0036,
-      "reward": 0.27719298852117435,
-      "reward_std": 0.33377805352211,
-      "rewards/accuracy_reward": 0.27719298852117435,
+      "loss": 0.0,
+      "reward": 0.7456140472700722,
+      "reward_std": 0.27623014795152767,
+      "rewards/accuracy_reward": 0.7456140472700722,
       "rewards/format_reward": 0.0,
       "step": 225
     },
     {
-      "completion_length": 413.22983430561266,
+      "completion_length": 541.6929962158204,
       "epoch": 0.5826666666666667,
-      "grad_norm": 0.5406464338302612,
-      "kl": 0.07656506990131579,
+      "grad_norm": 0.0,
+      "kl": 0.009119616056743421,
       "learning_rate": 1.3273298047249756e-06,
-      "loss": 0.0031,
-      "reward": 0.3421052706869025,
-      "reward_std": 0.40586905918623273,
-      "rewards/accuracy_reward": 0.3421052706869025,
+      "loss": 0.0,
+      "reward": 0.7964912389454089,
+      "reward_std": 0.2623067799367403,
+      "rewards/accuracy_reward": 0.7964912389454089,
       "rewards/format_reward": 0.0,
       "step": 230
     },
     {
-      "completion_length": 391.9754479659231,
+      "completion_length": 555.8666826750102,
       "epoch": 0.5953333333333334,
-      "grad_norm": 0.6628542542457581,
-      "kl": 0.07627788342927631,
+      "grad_norm": 0.1013207659125328,
+      "kl": 0.008411447625411184,
       "learning_rate": 1.2614047559713923e-06,
-      "loss": 0.0031,
-      "reward": 0.47192983580263037,
-      "reward_std": 0.3767852387930218,
-      "rewards/accuracy_reward": 0.47192983580263037,
+      "loss": 0.0,
+      "reward": 0.8263158042179911,
+      "reward_std": 0.25948601145493355,
+      "rewards/accuracy_reward": 0.8263158042179911,
       "rewards/format_reward": 0.0,
       "step": 235
     },
     {
-      "completion_length": 398.3614137348376,
+      "completion_length": 564.3929960552015,
       "epoch": 0.608,
-      "grad_norm": 0.22894343733787537,
-      "kl": 0.09133493523848685,
+      "grad_norm": 0.08883073180913925,
+      "kl": 0.008549579821134868,
       "learning_rate": 1.1959494106708598e-06,
-      "loss": 0.0037,
-      "reward": 0.4543859756306598,
-      "reward_std": 0.41446375846862793,
-      "rewards/accuracy_reward": 0.4543859756306598,
+      "loss": 0.0,
+      "reward": 0.792982468950121,
+      "reward_std": 0.25643417960719056,
+      "rewards/accuracy_reward": 0.792982468950121,
       "rewards/format_reward": 0.0,
       "step": 240
     },
     {
-      "completion_length": 382.73158842387954,
+      "completion_length": 536.9596663625617,
       "epoch": 0.6206666666666667,
-      "grad_norm": 0.2360672801733017,
-      "kl": 0.12881951583059212,
+      "grad_norm": 0.04451132193207741,
+      "kl": 0.009548789576480263,
       "learning_rate": 1.1310926255482204e-06,
-      "loss": 0.0052,
-      "reward": 0.5070175553622999,
-      "reward_std": 0.38596598882424205,
-      "rewards/accuracy_reward": 0.5070175553622999,
+      "loss": 0.0,
+      "reward": 0.8052631693451028,
+      "reward_std": 0.24199818247242977,
+      "rewards/accuracy_reward": 0.8052631693451028,
       "rewards/format_reward": 0.0,
       "step": 245
     },
     {
-      "completion_length": 379.06843085038037,
+      "completion_length": 544.7193134508635,
       "epoch": 0.6333333333333333,
-      "grad_norm": 0.41622936725616455,
-      "kl": 0.4508557771381579,
+      "grad_norm": 0.0,
+      "kl": 0.009332596628289473,
       "learning_rate": 1.0669620789905688e-06,
-      "loss": 0.018,
-      "reward": 0.5122807151392886,
-      "reward_std": 0.35713848534383275,
-      "rewards/accuracy_reward": 0.5122807151392886,
+      "loss": 0.0,
+      "reward": 0.800000010038677,
+      "reward_std": 0.23219164547167326,
+      "rewards/accuracy_reward": 0.800000010038677,
       "rewards/format_reward": 0.0,
       "step": 250
     },
     {
-      "completion_length": 379.2842171116879,
+      "completion_length": 552.8649261474609,
       "epoch": 0.646,
-      "grad_norm": 2.963742256164551,
-      "kl": 1.111646792763158,
+      "grad_norm": 0.06963980942964554,
+      "kl": 0.009455148797286185,
       "learning_rate": 1.0036840196969795e-06,
-      "loss": 0.0445,
-      "reward": 0.5614035189151764,
-      "reward_std": 0.33363121528374523,
-      "rewards/accuracy_reward": 0.5614035189151764,
+      "loss": 0.0,
+      "reward": 0.807017553009485,
+      "reward_std": 0.21750171090427198,
+      "rewards/accuracy_reward": 0.807017553009485,
       "rewards/format_reward": 0.0,
       "step": 255
     },
     {
-      "completion_length": 390.0175530684622,
+      "completion_length": 560.9298404091282,
       "epoch": 0.6586666666666666,
-      "grad_norm": 1.9537420272827148,
-      "kl": 1.1658845600328946,
+      "grad_norm": 0.13915280997753143,
+      "kl": 0.008615754780016447,
       "learning_rate": 9.413830181427508e-07,
-      "loss": 0.0466,
-      "reward": 0.5263158009240502,
-      "reward_std": 0.3237152021182211,
-      "rewards/accuracy_reward": 0.5263158009240502,
+      "loss": 0.0,
+      "reward": 0.7894736975431442,
+      "reward_std": 0.26591097994854573,
+      "rewards/accuracy_reward": 0.7894736975431442,
       "rewards/format_reward": 0.0,
       "step": 260
     },
     {
-      "completion_length": 406.68421936035156,
+      "completion_length": 583.3000167043585,
       "epoch": 0.6713333333333333,
-      "grad_norm": 0.1643250286579132,
-      "kl": 9.794161184210527,
+      "grad_norm": 0.0,
+      "kl": 0.008441322728207237,
       "learning_rate": 8.801817213474331e-07,
-      "loss": 0.3924,
-      "reward": 0.44210527413769773,
-      "reward_std": 0.3688186996861508,
-      "rewards/accuracy_reward": 0.44210527413769773,
+      "loss": 0.0,
+      "reward": 0.7508772039099744,
+      "reward_std": 0.2809643321915677,
+      "rewards/accuracy_reward": 0.7508772039099744,
       "rewards/format_reward": 0.0,
       "step": 265
     },
     {
-      "completion_length": 394.60878087093954,
+      "completion_length": 531.2859802246094,
       "epoch": 0.684,
-      "grad_norm": 0.5662396550178528,
-      "kl": 0.29067961040296053,
+      "grad_norm": 0.05701286345720291,
+      "kl": 0.00885218570106908,
       "learning_rate": 8.202006114294044e-07,
-      "loss": 0.0116,
-      "reward": 0.47368422009442984,
-      "reward_std": 0.37373340569044416,
-      "rewards/accuracy_reward": 0.47368422009442984,
+      "loss": 0.0,
+      "reward": 0.7877193099574039,
+      "reward_std": 0.2481692119648582,
+      "rewards/accuracy_reward": 0.7877193099574039,
       "rewards/format_reward": 0.0,
       "step": 270
     },
     {
-      "completion_length": 383.15790437397203,
+      "completion_length": 531.2403676886307,
       "epoch": 0.6966666666666667,
-      "grad_norm": 0.36152008175849915,
-      "kl": 0.8754098992598685,
+      "grad_norm": 0.05670388042926788,
+      "kl": 0.009579307154605263,
       "learning_rate": 7.615577684223272e-07,
-      "loss": 0.035,
-      "reward": 0.5210526431861676,
-      "reward_std": 0.39000940040538185,
-      "rewards/accuracy_reward": 0.5210526431861676,
+      "loss": 0.0,
+      "reward": 0.8175438716223365,
+      "reward_std": 0.23557321397881759,
+      "rewards/accuracy_reward": 0.8175438716223365,
       "rewards/format_reward": 0.0,
       "step": 275
     },
     {
-      "completion_length": 396.80176439787215,
+      "completion_length": 544.3842264275802,
       "epoch": 0.7093333333333334,
-      "grad_norm": 1.2584236860275269,
-      "kl": 1.3499331825657894,
+      "grad_norm": 0.0,
+      "kl": 0.009712299547697369,
       "learning_rate": 7.043686378203864e-07,
-      "loss": 0.054,
-      "reward": 0.4596491366624832,
-      "reward_std": 0.37697656687937287,
-      "rewards/accuracy_reward": 0.4596491366624832,
+      "loss": 0.0,
+      "reward": 0.7701754490011616,
+      "reward_std": 0.25272287663660553,
+      "rewards/accuracy_reward": 0.7701754490011616,
       "rewards/format_reward": 0.0,
       "step": 280
     },
     {
-      "completion_length": 408.31755178351153,
+      "completion_length": 571.7438746402138,
       "epoch": 0.722,
-      "grad_norm": 1.6444029808044434,
-      "kl": 0.37659269634046055,
+      "grad_norm": 0.08127343654632568,
+      "kl": 0.008696064196134869,
       "learning_rate": 6.487458033099425e-07,
-      "loss": 0.0151,
-      "reward": 0.43333334467912976,
-      "reward_std": 0.41270798601602254,
-      "rewards/accuracy_reward": 0.43333334467912976,
+      "loss": 0.0,
+      "reward": 0.7701754521382482,
+      "reward_std": 0.2902606151605907,
+      "rewards/accuracy_reward": 0.7701754521382482,
       "rewards/format_reward": 0.0,
       "step": 285
     },
     {
-      "completion_length": 396.7526412160773,
+      "completion_length": 532.1596609015214,
       "epoch": 0.7346666666666667,
-      "grad_norm": 0.24537613987922668,
-      "kl": 0.7064022666529606,
+      "grad_norm": 0.072137750685215,
+      "kl": 0.008551828484786184,
       "learning_rate": 5.947987651349942e-07,
-      "loss": 0.0283,
-      "reward": 0.5245614156911248,
-      "reward_std": 0.39676410242130883,
-      "rewards/accuracy_reward": 0.5245614156911248,
+      "loss": 0.0,
+      "reward": 0.8070175544211738,
+      "reward_std": 0.21213936178307785,
+      "rewards/accuracy_reward": 0.8070175544211738,
       "rewards/format_reward": 0.0,
       "step": 290
     },
     {
-      "completion_length": 396.04386933979237,
+      "completion_length": 535.1210667660362,
       "epoch": 0.7473333333333333,
-      "grad_norm": 1.1248526573181152,
-      "kl": 2.2961811266447367,
+      "grad_norm": 0.07434926927089691,
+      "kl": 0.008226896587171052,
       "learning_rate": 5.426337245327703e-07,
-      "loss": 0.0919,
-      "reward": 0.5070175574014061,
-      "reward_std": 0.3869973038372241,
-      "rewards/accuracy_reward": 0.5070175574014061,
+      "loss": 0.0,
+      "reward": 0.8157894832523246,
+      "reward_std": 0.22407589335190622,
+      "rewards/accuracy_reward": 0.8157894832523246,
       "rewards/format_reward": 0.0,
       "step": 295
     },
     {
-      "completion_length": 393.7561492919922,
+      "completion_length": 536.6631729929071,
       "epoch": 0.76,
-      "grad_norm": 5.84006929397583,
-      "kl": 5.933592465049342,
+      "grad_norm": 0.08616889268159866,
+      "kl": 0.010335339997944079,
       "learning_rate": 4.923533746638108e-07,
-      "loss": 0.2375,
-      "reward": 0.48245614986670643,
-      "reward_std": 0.40506037599162054,
-      "rewards/accuracy_reward": 0.48245614986670643,
+      "loss": 0.0,
+      "reward": 0.8105263270829853,
+      "reward_std": 0.24915841535518043,
+      "rewards/accuracy_reward": 0.8105263270829853,
       "rewards/format_reward": 0.0,
       "step": 300
     },
     {
       "epoch": 0.76,
-      "eval_completion_length": 393.75374295043946,
-      "eval_kl": 3.580558123779297,
-      "eval_loss": 0.14329032599925995,
-      "eval_reward": 0.41980000983178617,
-      "eval_reward_std": 0.3639850652396679,
-      "eval_rewards/accuracy_reward": 0.41980000983178617,
+      "eval_completion_length": 545.0558823242187,
+      "eval_kl": 0.6695549743652344,
+      "eval_loss": 3.003481197083602e-08,
+      "eval_reward": 0.6868333460628986,
+      "eval_reward_std": 0.3212484232842922,
+      "eval_rewards/accuracy_reward": 0.6868333460628986,
       "eval_rewards/format_reward": 0.0,
-      "eval_runtime": 47686.4832,
-      "eval_samples_per_second": 0.105,
-      "eval_steps_per_second": 0.017,
+      "eval_runtime": 75622.6169,
+      "eval_samples_per_second": 0.066,
+      "eval_steps_per_second": 0.011,
       "step": 300
     },
     {
-      "completion_length": 406.72983366313736,
+      "completion_length": 574.1017703407689,
       "epoch": 0.7726666666666666,
-      "grad_norm": 0.570837140083313,
-      "kl": 2.009347373560855,
+      "grad_norm": 0.0810866504907608,
+      "kl": 0.02553052400287829,
       "learning_rate": 4.440566984481256e-07,
-      "loss": 0.0804,
-      "reward": 0.42280702669369546,
-      "reward_std": 0.4078474631434993,
-      "rewards/accuracy_reward": 0.42280702669369546,
+      "loss": 0.0,
+      "reward": 0.7666666799469998,
+      "reward_std": 0.27946487539692927,
+      "rewards/accuracy_reward": 0.7666666799469998,
       "rewards/format_reward": 0.0,
       "step": 305
     },
     {
-      "completion_length": 387.854396458676,
+      "completion_length": 550.2579098350124,
       "epoch": 0.7853333333333333,
-      "grad_norm": 0.4837806224822998,
-      "kl": 0.8569181743421053,
+      "grad_norm": 0.09656031429767609,
+      "kl": 0.009140657123766447,
       "learning_rate": 3.978387737053994e-07,
-      "loss": 0.0343,
-      "reward": 0.5350877289709292,
-      "reward_std": 0.3191217996572193,
-      "rewards/accuracy_reward": 0.5350877289709292,
+      "loss": 0.0,
+      "reward": 0.7824561517489584,
+      "reward_std": 0.23484032687387968,
+      "rewards/accuracy_reward": 0.7824561517489584,
       "rewards/format_reward": 0.0,
       "step": 310
     },
     {
-      "completion_length": 391.2175537109375,
+      "completion_length": 569.4912441855982,
       "epoch": 0.798,
-      "grad_norm": 1.2316937446594238,
-      "kl": 1.243909333881579,
+      "grad_norm": 0.094791479408741,
+      "kl": 0.009346088610197369,
       "learning_rate": 3.5379058598286167e-07,
-      "loss": 0.0497,
-      "reward": 0.4842105376093011,
-      "reward_std": 0.3719776351200907,
-      "rewards/accuracy_reward": 0.4842105376093011,
+      "loss": 0.0,
+      "reward": 0.7666666777510392,
+      "reward_std": 0.25904074530852467,
+      "rewards/accuracy_reward": 0.7666666777510392,
       "rewards/format_reward": 0.0,
       "step": 315
     },
     {
-      "completion_length": 404.43509698165093,
+      "completion_length": 579.2614203202097,
       "epoch": 0.8106666666666666,
-      "grad_norm": 1.2691428661346436,
-      "kl": 1.967847964638158,
+      "grad_norm": 0.10730049759149551,
+      "kl": 4.134726434004934,
       "learning_rate": 3.119988494392894e-07,
-      "loss": 0.0787,
-      "reward": 0.47543860705275287,
-      "reward_std": 0.40146460689996416,
-      "rewards/accuracy_reward": 0.47543860705275287,
+      "loss": 0.0,
+      "reward": 0.7508772053216634,
+      "reward_std": 0.2928671808619248,
+      "rewards/accuracy_reward": 0.7508772053216634,
       "rewards/format_reward": 0.0,
       "step": 320
     },
     {
-      "completion_length": 385.6350979453639,
+      "completion_length": 556.2263304057874,
       "epoch": 0.8233333333333334,
-      "grad_norm": 2.1663596630096436,
-      "kl": 1.7478567023026317,
+      "grad_norm": 0.07251843065023422,
+      "kl": 0.009259033203125,
       "learning_rate": 2.725458361377465e-07,
-      "loss": 0.0699,
-      "reward": 0.5333333443654211,
-      "reward_std": 0.3384051385678743,
-      "rewards/accuracy_reward": 0.5333333443654211,
+      "loss": 0.0,
+      "reward": 0.7736842222903905,
+      "reward_std": 0.24773237705230713,
+      "rewards/accuracy_reward": 0.7736842222903905,
       "rewards/format_reward": 0.0,
       "step": 325
     },
     {
-      "completion_length": 389.51755499588813,
+      "completion_length": 537.1491384405839,
       "epoch": 0.836,
-      "grad_norm": 0.9927846789360046,
-      "kl": 2.0717015316611844,
+      "grad_norm": 0.0,
+      "kl": 0.009849789268092105,
       "learning_rate": 2.3550921408312737e-07,
-      "loss": 0.0829,
-      "reward": 0.5052631704430831,
-      "reward_std": 0.386816783641514,
-      "rewards/accuracy_reward": 0.5052631704430831,
+      "loss": 0.0,
+      "reward": 0.8140350983331078,
+      "reward_std": 0.23072349554614016,
+      "rewards/accuracy_reward": 0.8140350983331078,
       "rewards/format_reward": 0.0,
       "step": 330
     },
     {
-      "completion_length": 380.80000770970395,
+      "completion_length": 547.5140491686369,
       "epoch": 0.8486666666666667,
-      "grad_norm": 1.9131776094436646,
-      "kl": 1.4728098016036184,
+      "grad_norm": 0.08143898099660873,
+      "kl": 0.010343531558388158,
       "learning_rate": 2.0096189432334195e-07,
-      "loss": 0.059,
-      "reward": 0.46491229016529884,
-      "reward_std": 0.3830212574256094,
-      "rewards/accuracy_reward": 0.46491229016529884,
+      "loss": 0.0,
+      "reward": 0.7824561493961435,
+      "reward_std": 0.23674531171196386,
+      "rewards/accuracy_reward": 0.7824561493961435,
       "rewards/format_reward": 0.0,
       "step": 335
     },
     {
-      "completion_length": 384.0947445518092,
+      "completion_length": 541.5175582082649,
       "epoch": 0.8613333333333333,
-      "grad_norm": 0.5461324453353882,
-      "kl": 1.2412668328536185,
+      "grad_norm": 0.15802732110023499,
+      "kl": 0.009666683799342105,
       "learning_rate": 1.6897188741514286e-07,
-      "loss": 0.0497,
-      "reward": 0.46842106282711027,
-      "reward_std": 0.372008940106944,
-      "rewards/accuracy_reward": 0.46842106282711027,
+      "loss": 0.0,
+      "reward": 0.7666666757119329,
+      "reward_std": 0.24941472881718685,
+      "rewards/accuracy_reward": 0.7666666757119329,
       "rewards/format_reward": 0.0,
       "step": 340
     },
     {
-      "completion_length": 395.47720272666527,
+      "completion_length": 566.2824713456004,
       "epoch": 0.874,
-      "grad_norm": 0.5957249999046326,
-      "kl": 1.3121890419407896,
+      "grad_norm": 0.08206634968519211,
+      "kl": 0.009324886924342106,
       "learning_rate": 1.396021695371582e-07,
-      "loss": 0.0525,
-      "reward": 0.4543859761012228,
-      "reward_std": 0.35364981732870404,
-      "rewards/accuracy_reward": 0.4543859761012228,
+      "loss": 0.0,
+      "reward": 0.7543859756306598,
+      "reward_std": 0.23858530960584942,
+      "rewards/accuracy_reward": 0.7543859756306598,
       "rewards/format_reward": 0.0,
       "step": 345
     },
     {
-      "completion_length": 388.5894830001028,
+      "completion_length": 567.8473815917969,
       "epoch": 0.8866666666666667,
-      "grad_norm": 0.7833815813064575,
-      "kl": 1.2672890111019737,
+      "grad_norm": 0.06388476490974426,
+      "kl": 0.010038034539473684,
       "learning_rate": 1.1291055851370623e-07,
-      "loss": 0.0507,
-      "reward": 0.46491229063586187,
-      "reward_std": 0.3626813524647763,
-      "rewards/accuracy_reward": 0.46491229063586187,
+      "loss": 0.0,
+      "reward": 0.7491228202455922,
+      "reward_std": 0.2948876986378118,
+      "rewards/accuracy_reward": 0.7491228202455922,
       "rewards/format_reward": 0.0,
       "step": 350
     },
     {
-      "completion_length": 393.6350974635074,
+      "completion_length": 576.67545663934,
       "epoch": 0.8993333333333333,
-      "grad_norm": 1.0711956024169922,
-      "kl": 1.032947419819079,
+      "grad_norm": 0.058260105550289154,
+      "kl": 0.008216456363075659,
       "learning_rate": 8.894959999345015e-08,
-      "loss": 0.0413,
-      "reward": 0.4210526403627898,
-      "reward_std": 0.3503103620127628,
-      "rewards/accuracy_reward": 0.4210526403627898,
+      "loss": 0.0,
+      "reward": 0.8017543960558741,
+      "reward_std": 0.23333006344343488,
+      "rewards/accuracy_reward": 0.8017543960558741,
       "rewards/format_reward": 0.0,
       "step": 355
     },
     {
-      "completion_length": 389.8333436664782,
+      "completion_length": 549.091246434262,
       "epoch": 0.912,
-      "grad_norm": 0.6254836320877075,
-      "kl": 0.9737092670641447,
+      "grad_norm": 0.10874364525079727,
+      "kl": 0.00869959781044408,
       "learning_rate": 6.776646400696212e-08,
-      "loss": 0.039,
-      "reward": 0.4859649231559352,
-      "reward_std": 0.38237022004629434,
-      "rewards/accuracy_reward": 0.4859649231559352,
+      "loss": 0.0,
+      "reward": 0.796491244749019,
+      "reward_std": 0.3010250455454776,
+      "rewards/accuracy_reward": 0.796491244749019,
       "rewards/format_reward": 0.0,
       "step": 360
     },
     {
-      "completion_length": 389.48948348195927,
+      "completion_length": 560.8719421386719,
       "epoch": 0.9246666666666666,
-      "grad_norm": 1.6106293201446533,
-      "kl": 0.9006559673108553,
+      "grad_norm": 0.10691142827272415,
+      "kl": 0.009839509662828948,
       "learning_rate": 4.940285210684375e-08,
-      "loss": 0.036,
-      "reward": 0.49122808324663264,
-      "reward_std": 0.3813075975367897,
-      "rewards/accuracy_reward": 0.49122808324663264,
+      "loss": 0.0,
+      "reward": 0.7964912431804757,
+      "reward_std": 0.28669615042837043,
+      "rewards/accuracy_reward": 0.7964912431804757,
       "rewards/format_reward": 0.0,
       "step": 365
     },
     {
-      "completion_length": 387.6245711477179,
+      "completion_length": 570.056158126028,
       "epoch": 0.9373333333333334,
-      "grad_norm": 2.270439863204956,
-      "kl": 1.5312891909950659,
+      "grad_norm": 0.14333242177963257,
+      "kl": 0.008819580078125,
       "learning_rate": 3.389491527319999e-08,
-      "loss": 0.0613,
-      "reward": 0.4543859757875141,
-      "reward_std": 0.3992635683009499,
-      "rewards/accuracy_reward": 0.4543859757875141,
+      "loss": 0.0,
+      "reward": 0.7631579105791293,
+      "reward_std": 0.3113442129210422,
+      "rewards/accuracy_reward": 0.7631579105791293,
       "rewards/format_reward": 0.0,
       "step": 370
     },
     {
-      "completion_length": 398.4315892269737,
+      "completion_length": 581.7122971384149,
       "epoch": 0.95,
-      "grad_norm": 2.971569061279297,
-      "kl": 1.466099146792763,
+      "grad_norm": 0.12897399067878723,
+      "kl": 0.009299830386513159,
       "learning_rate": 2.127318274608381e-08,
-      "loss": 0.0586,
-      "reward": 0.4473684310913086,
-      "reward_std": 0.30884474672769247,
-      "rewards/accuracy_reward": 0.4473684310913086,
+      "loss": 0.0,
+      "reward": 0.759649133211688,
+      "reward_std": 0.24736052971137198,
+      "rewards/accuracy_reward": 0.759649133211688,
       "rewards/format_reward": 0.0,
       "step": 375
     },
     {
-      "completion_length": 382.3105351498252,
+      "completion_length": 519.6631728322883,
       "epoch": 0.9626666666666667,
-      "grad_norm": 0.4336649775505066,
-      "kl": 1.0901534231085526,
+      "grad_norm": 0.046151068061590195,
+      "kl": 0.018807340923108554,
       "learning_rate": 1.1562501925013125e-08,
-      "loss": 0.0436,
-      "reward": 0.5385965030444296,
-      "reward_std": 0.3790620725405844,
-      "rewards/accuracy_reward": 0.5385965030444296,
+      "loss": 0.0,
+      "reward": 0.8105263286515286,
+      "reward_std": 0.24725342863484434,
+      "rewards/accuracy_reward": 0.8105263286515286,
       "rewards/format_reward": 0.0,
       "step": 380
     },
     {
-      "completion_length": 382.67018448679073,
+      "completion_length": 522.570188984118,
       "epoch": 0.9753333333333334,
-      "grad_norm": 2.1975314617156982,
-      "kl": 1.1284366005345394,
+      "grad_norm": 0.10620765388011932,
+      "kl": 0.00986279939350329,
       "learning_rate": 4.781989453874814e-09,
-      "loss": 0.0452,
-      "reward": 0.5087719407520797,
-      "reward_std": 0.3483945708525808,
-      "rewards/accuracy_reward": 0.5087719407520797,
+      "loss": 0.0,
+      "reward": 0.7964912391022632,
+      "reward_std": 0.2455626440675635,
+      "rewards/accuracy_reward": 0.7964912391022632,
       "rewards/format_reward": 0.0,
       "step": 385
     },
     {
-      "completion_length": 395.0982547157689,
+      "completion_length": 536.8315922787315,
       "epoch": 0.988,
-      "grad_norm": 1.3517661094665527,
-      "kl": 1.2858867444490132,
+      "grad_norm": 0.09291122853755951,
+      "kl": 0.010211502878289473,
       "learning_rate": 9.44993587509657e-10,
-      "loss": 0.0514,
-      "reward": 0.5070175545780282,
-      "reward_std": 0.4019014430673499,
-      "rewards/accuracy_reward": 0.5070175545780282,
+      "loss": 0.0,
+      "reward": 0.8070175572445518,
+      "reward_std": 0.2619433641433716,
+      "rewards/accuracy_reward": 0.8070175572445518,
       "rewards/format_reward": 0.0,
       "step": 390
     },
     {
-      "completion_length": 403.1096580906918,
+      "completion_length": 567.2982639513517,
       "epoch": 0.9981333333333333,
-      "kl": 2.4145796926398027,
-      "reward": 0.42324562468811083,
-      "reward_std": 0.32168745288723394,
-      "rewards/accuracy_reward": 0.42324562468811083,
+      "kl": 0.00917715775339227,
+      "reward": 0.7828947472337046,
+      "reward_std": 0.2575972625299504,
+      "rewards/accuracy_reward": 0.7828947472337046,
       "rewards/format_reward": 0.0,
       "step": 394,
       "total_flos": 0.0,
-      "train_loss": 0.027036830155200475,
-      "train_runtime": 281082.0852,
-      "train_samples_per_second": 0.027,
+      "train_loss": 0.0008256121124141636,
+      "train_runtime": 419098.9196,
+      "train_samples_per_second": 0.018,
       "train_steps_per_second": 0.001
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6f942f0096b55244d30974e5a2a781aa646e662406e9dee451d546238cbf9b6c
+oid sha256:84f3f7010102b0e305468c1cf34da3672c32048c60d1693fa81fc6817dc89fe5
 size 7480