Model save

README.md

@@ -27,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/jzhang-x-01/huggingface/runs/f17te69y) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/jzhang-x-01/huggingface/runs/l1ih86fe) 
 
 
 This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.00041823586402218955,
-    "train_runtime": 8141.2102,
+    "train_loss": 0.002154490305616204,
+    "train_runtime": 8228.6792,
     "train_samples": 7500,
-    "train_samples_per_second": 0.921,
+    "train_samples_per_second": 0.911,
     "train_steps_per_second": 0.007
 }

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d372229b5765122f0f665a6bc9ea66c8e501cade1deea988f753f4546d8c01e2
+oid sha256:da4320c5ba6d132b13e0697de200379e6696bb453e297dd29a845b6cf0a5ad11
 size 4877660776

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cdf88cc467578d4d66756ce7b0778b49989445997baecfa542f50a7df748a535
+oid sha256:eb5180786ab5e7eb5c36cbfc486ba5d1a87942134f42b637939668da6e4b7b68
 size 4932751008

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fb9ae13bc1ac27d2fe51b43d97591e1d9762abde6837c547e2c52d5b61fca3bd
+oid sha256:1c5df4aabc965da16054c18701d906c2afcfea880192d0a5ac4c9759ba58ae05
 size 4330865200

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ed2f6405de1c7e3a363f3541d86a91cc532a81425d2727016b466c48f17be4a9
+oid sha256:aac5bbf08545411482d72b15e3233be68a3e299b0f204d9e15b514f68fa374df
 size 1089994880

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.00041823586402218955,
-    "train_runtime": 8141.2102,
+    "train_loss": 0.002154490305616204,
+    "train_runtime": 8228.6792,
     "train_samples": 7500,
-    "train_samples_per_second": 0.921,
+    "train_samples_per_second": 0.911,
     "train_steps_per_second": 0.007
 }

trainer_state.json

@@ -9,9 +9,9 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 606.4564895629883,
+      "completion_length": 606.5457763671875,
       "epoch": 0.017057569296375266,
-      "grad_norm": 5.349207388441972,
+      "grad_norm": 5.351304250136567,
       "kl": 0.0,
       "learning_rate": 5e-07,
       "loss": 0.0,
@@ -22,161 +22,161 @@
       "step": 1
     },
     {
-      "completion_length": 603.4832859039307,
+      "completion_length": 607.6144218444824,
       "epoch": 0.08528784648187633,
-      "grad_norm": 9.588701812058716,
-      "kl": 0.00046131014823913574,
+      "grad_norm": 11.35565102634353,
+      "kl": 0.00033593177795410156,
       "learning_rate": 2.5e-06,
       "loss": 0.0,
-      "reward": 0.6004464514553547,
-      "reward_std": 0.3618758851662278,
-      "rewards/accuracy_reward": 0.6004464514553547,
+      "reward": 0.6026785979047418,
+      "reward_std": 0.3518537702038884,
+      "rewards/accuracy_reward": 0.6026785979047418,
       "rewards/format_reward": 0.0,
       "step": 5
     },
     {
-      "completion_length": 620.3982391357422,
+      "completion_length": 622.7357368469238,
       "epoch": 0.17057569296375266,
-      "grad_norm": 0.4408238121081561,
-      "kl": 0.016131591796875,
+      "grad_norm": 1.9080536354821958,
+      "kl": 0.004211950302124024,
       "learning_rate": 2.956412726139078e-06,
-      "loss": 0.0006,
-      "reward": 0.6678571753203869,
-      "reward_std": 0.3235505763441324,
-      "rewards/accuracy_reward": 0.6678571753203869,
+      "loss": 0.0002,
+      "reward": 0.6660714566707611,
+      "reward_std": 0.31057145930826663,
+      "rewards/accuracy_reward": 0.6660714566707611,
       "rewards/format_reward": 0.0,
       "step": 10
     },
     {
-      "completion_length": 636.4580642700196,
+      "completion_length": 631.8491333007812,
       "epoch": 0.255863539445629,
-      "grad_norm": 70.0216466198137,
-      "kl": 0.021285057067871094,
+      "grad_norm": 0.20168403792141756,
+      "kl": 0.007109832763671875,
       "learning_rate": 2.7836719084521715e-06,
-      "loss": 0.0009,
-      "reward": 0.7464286029338837,
-      "reward_std": 0.24510460048913957,
-      "rewards/accuracy_reward": 0.7464286029338837,
+      "loss": 0.0003,
+      "reward": 0.7424107462167739,
+      "reward_std": 0.2543599892407656,
+      "rewards/accuracy_reward": 0.7424107462167739,
       "rewards/format_reward": 0.0,
       "step": 15
     },
     {
-      "completion_length": 617.2065002441407,
+      "completion_length": 622.2803810119628,
       "epoch": 0.3411513859275053,
-      "grad_norm": 0.12452084814574951,
-      "kl": 0.004275131225585938,
+      "grad_norm": 0.495428185088676,
+      "kl": 0.02277984619140625,
       "learning_rate": 2.4946839873611927e-06,
-      "loss": 0.0002,
-      "reward": 0.752678605914116,
-      "reward_std": 0.22502579726278782,
-      "rewards/accuracy_reward": 0.752678605914116,
+      "loss": 0.0009,
+      "reward": 0.7529018178582192,
+      "reward_std": 0.23195756375789642,
+      "rewards/accuracy_reward": 0.7529018178582192,
       "rewards/format_reward": 0.0,
       "step": 20
     },
     {
-      "completion_length": 620.6533752441406,
+      "completion_length": 629.6663253784179,
       "epoch": 0.42643923240938164,
-      "grad_norm": 0.4497624610789142,
-      "kl": 0.006324386596679688,
+      "grad_norm": 0.2077076072198444,
+      "kl": 0.0070018768310546875,
       "learning_rate": 2.1156192081791355e-06,
       "loss": 0.0003,
-      "reward": 0.7526786029338837,
-      "reward_std": 0.20553307328373194,
-      "rewards/accuracy_reward": 0.7526786029338837,
+      "reward": 0.7497768178582191,
+      "reward_std": 0.22464404683560132,
+      "rewards/accuracy_reward": 0.7497768178582191,
       "rewards/format_reward": 0.0,
       "step": 25
     },
     {
-      "completion_length": 617.4848472595215,
+      "completion_length": 628.7462333679199,
       "epoch": 0.511727078891258,
-      "grad_norm": 7.695959344352079,
-      "kl": 0.014894866943359375,
+      "grad_norm": 223.6982144990451,
+      "kl": 0.3876213073730469,
       "learning_rate": 1.6808050203829845e-06,
-      "loss": 0.0006,
-      "reward": 0.7448661029338837,
-      "reward_std": 0.19868585970252753,
-      "rewards/accuracy_reward": 0.7448661029338837,
+      "loss": 0.0155,
+      "reward": 0.7379464656114578,
+      "reward_std": 0.21265463214367628,
+      "rewards/accuracy_reward": 0.7379464656114578,
       "rewards/format_reward": 0.0,
       "step": 30
     },
     {
-      "completion_length": 612.6093978881836,
+      "completion_length": 614.8966758728027,
       "epoch": 0.5970149253731343,
-      "grad_norm": 0.3012279035059688,
-      "kl": 0.031812286376953124,
+      "grad_norm": 2.3665634575263748,
+      "kl": 0.005752182006835938,
       "learning_rate": 1.2296174432791415e-06,
-      "loss": 0.0013,
-      "reward": 0.739732176065445,
-      "reward_std": 0.19879167079925536,
-      "rewards/accuracy_reward": 0.739732176065445,
+      "loss": 0.0002,
+      "reward": 0.7314732424914837,
+      "reward_std": 0.19922155924141408,
+      "rewards/accuracy_reward": 0.7314732424914837,
       "rewards/format_reward": 0.0,
       "step": 35
     },
     {
-      "completion_length": 597.8221229553222,
+      "completion_length": 603.5955581665039,
       "epoch": 0.6823027718550106,
-      "grad_norm": 0.13735068646343107,
-      "kl": 0.00665283203125,
+      "grad_norm": 0.41861750967507205,
+      "kl": 0.007678604125976563,
       "learning_rate": 8.029152419343472e-07,
       "loss": 0.0003,
-      "reward": 0.7627232506871223,
-      "reward_std": 0.20173386242240668,
-      "rewards/accuracy_reward": 0.7627232506871223,
+      "reward": 0.7553571775555611,
+      "reward_std": 0.20080968737602234,
+      "rewards/accuracy_reward": 0.7553571775555611,
       "rewards/format_reward": 0.0,
       "step": 40
     },
     {
-      "completion_length": 607.5589538574219,
+      "completion_length": 619.6895401000977,
       "epoch": 0.767590618336887,
-      "grad_norm": 0.09444476728427914,
-      "kl": 0.006229019165039063,
+      "grad_norm": 0.5526088012366769,
+      "kl": 0.007487106323242188,
       "learning_rate": 4.3933982822017883e-07,
-      "loss": 0.0002,
-      "reward": 0.752901816368103,
-      "reward_std": 0.19051024238578976,
-      "rewards/accuracy_reward": 0.752901816368103,
+      "loss": 0.0003,
+      "reward": 0.725000037252903,
+      "reward_std": 0.23073445297777653,
+      "rewards/accuracy_reward": 0.725000037252903,
       "rewards/format_reward": 0.0,
       "step": 45
     },
     {
-      "completion_length": 609.9201133728027,
+      "completion_length": 613.0727882385254,
       "epoch": 0.8528784648187633,
-      "grad_norm": 0.13825347710108976,
-      "kl": 0.005415916442871094,
+      "grad_norm": 1.1048247702884233,
+      "kl": 0.00660247802734375,
       "learning_rate": 1.718159615201853e-07,
-      "loss": 0.0002,
-      "reward": 0.7462053894996643,
-      "reward_std": 0.19213472940027715,
-      "rewards/accuracy_reward": 0.7462053894996643,
+      "loss": 0.0003,
+      "reward": 0.7332589656114579,
+      "reward_std": 0.19824975840747355,
+      "rewards/accuracy_reward": 0.7332589656114579,
       "rewards/format_reward": 0.0,
       "step": 50
     },
     {
-      "completion_length": 603.0685539245605,
+      "completion_length": 608.9152099609375,
       "epoch": 0.9381663113006397,
-      "grad_norm": 0.11004891600095792,
-      "kl": 0.00497283935546875,
+      "grad_norm": 0.688326952525324,
+      "kl": 0.006847000122070313,
       "learning_rate": 2.4570139579284723e-08,
-      "loss": 0.0002,
-      "reward": 0.7816964641213417,
-      "reward_std": 0.19411077070981264,
-      "rewards/accuracy_reward": 0.7816964641213417,
+      "loss": 0.0003,
+      "reward": 0.7656250342726707,
+      "reward_std": 0.21631054822355508,
+      "rewards/accuracy_reward": 0.7656250342726707,
       "rewards/format_reward": 0.0,
       "step": 55
     },
     {
-      "completion_length": 591.0379651387533,
+      "completion_length": 603.1421381632487,
       "epoch": 0.9893390191897654,
-      "kl": 0.0048402150472005205,
-      "reward": 0.7686012263099352,
-      "reward_std": 0.16828599898144603,
-      "rewards/accuracy_reward": 0.7686012263099352,
+      "kl": 0.0070037841796875,
+      "reward": 0.751860149204731,
+      "reward_std": 0.20624662407984337,
+      "rewards/accuracy_reward": 0.751860149204731,
       "rewards/format_reward": 0.0,
       "step": 58,
       "total_flos": 0.0,
-      "train_loss": 0.00041823586402218955,
-      "train_runtime": 8141.2102,
-      "train_samples_per_second": 0.921,
+      "train_loss": 0.002154490305616204,
+      "train_runtime": 8228.6792,
+      "train_samples_per_second": 0.911,
       "train_steps_per_second": 0.007
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4a102f47943bd94aa7ecc64e47c8daa56e3ea0e2ce38b52d8866da5a59dd6ffd
+oid sha256:095c9ce79540846bb0717018dbf2b9b1f9126d535c416a479f2bd2e2f09039f3
 size 7928