jlchen-c commited on
Commit
c81b0ae
·
verified ·
1 Parent(s): f5a87aa

Model save

Browse files
README.md CHANGED
@@ -1,11 +1,9 @@
1
  ---
2
  base_model: Qwen/Qwen2.5-Math-7B
3
- datasets: DigitalLearningGmbH/MATH-lighteval
4
  library_name: transformers
5
  model_name: Qwen-2.5-7B-Simple-RL
6
  tags:
7
  - generated_from_trainer
8
- - open-r1
9
  - trl
10
  - grpo
11
  licence: license
@@ -13,7 +11,7 @@ licence: license
13
 
14
  # Model Card for Qwen-2.5-7B-Simple-RL
15
 
16
- This model is a fine-tuned version of [Qwen/Qwen2.5-Math-7B](https://huggingface.co/Qwen/Qwen2.5-Math-7B) on the [DigitalLearningGmbH/MATH-lighteval](https://huggingface.co/datasets/DigitalLearningGmbH/MATH-lighteval) dataset.
17
  It has been trained using [TRL](https://github.com/huggingface/trl).
18
 
19
  ## Quick start
@@ -29,7 +27,7 @@ print(output["generated_text"])
29
 
30
  ## Training procedure
31
 
32
- [<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/jun-liang-chen-the-hong-kong-polytechnic-university/huggingface/runs/czqrqax3)
33
 
34
 
35
  This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).
 
1
  ---
2
  base_model: Qwen/Qwen2.5-Math-7B
 
3
  library_name: transformers
4
  model_name: Qwen-2.5-7B-Simple-RL
5
  tags:
6
  - generated_from_trainer
 
7
  - trl
8
  - grpo
9
  licence: license
 
11
 
12
  # Model Card for Qwen-2.5-7B-Simple-RL
13
 
14
+ This model is a fine-tuned version of [Qwen/Qwen2.5-Math-7B](https://huggingface.co/Qwen/Qwen2.5-Math-7B).
15
  It has been trained using [TRL](https://github.com/huggingface/trl).
16
 
17
  ## Quick start
 
27
 
28
  ## Training procedure
29
 
30
+ [<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/jun-liang-chen-the-hong-kong-polytechnic-university/huggingface/runs/ye8fls8s)
31
 
32
 
33
  This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).
all_results.json CHANGED
@@ -1,13 +1,8 @@
1
  {
2
- "eval_loss": 0.07094848901033401,
3
- "eval_runtime": 47796.4045,
4
- "eval_samples": 5000,
5
- "eval_samples_per_second": 0.105,
6
- "eval_steps_per_second": 0.017,
7
  "total_flos": 0.0,
8
- "train_loss": 0.027036830155200475,
9
- "train_runtime": 281082.0852,
10
  "train_samples": 7500,
11
- "train_samples_per_second": 0.027,
12
  "train_steps_per_second": 0.001
13
  }
 
1
  {
 
 
 
 
 
2
  "total_flos": 0.0,
3
+ "train_loss": 0.0008256121124141636,
4
+ "train_runtime": 419098.9196,
5
  "train_samples": 7500,
6
+ "train_samples_per_second": 0.018,
7
  "train_steps_per_second": 0.001
8
  }
config.json CHANGED
@@ -23,7 +23,7 @@
23
  "tie_word_embeddings": false,
24
  "torch_dtype": "bfloat16",
25
  "transformers_version": "4.49.0.dev0",
26
- "use_cache": true,
27
  "use_mrope": false,
28
  "use_sliding_window": false,
29
  "vocab_size": 152064
 
23
  "tie_word_embeddings": false,
24
  "torch_dtype": "bfloat16",
25
  "transformers_version": "4.49.0.dev0",
26
+ "use_cache": false,
27
  "use_mrope": false,
28
  "use_sliding_window": false,
29
  "vocab_size": 152064
model-00001-of-00004.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:42712c9569b2980599252a40db264d2d1c08c116a7d552ec7b5bdb051d949b2b
3
  size 4877660776
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3d2460834ab32758c7929fd2127f4764dda55d37b2ed228b7d65398fdd13aa3f
3
  size 4877660776
model-00002-of-00004.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:2e04adf4708f0377e2df413936055a264f94570ec1cf509b517212be76a1a036
3
  size 4932751008
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bcb46f0ac1dcead4a1ccca541f796c45a1d7d22a42f37e19cdcdf1b60bc0583c
3
  size 4932751008
model-00003-of-00004.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:fe17ddeef939048814b042e4aa1e4f9ab07697fa5ab028c15f65110342623024
3
  size 4330865200
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9ec9437e98f69f030c1e09f702858d79bfacf2ed020e66df9be24b59c445c214
3
  size 4330865200
model-00004-of-00004.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:f7dcdf332fdd5d2fb6991cc373e71db11897bebe7c65364593a931650149ca68
3
  size 1089994880
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fc740e9e6a6d8c353842a1ed8a64535fa9ce1baa0366313305dce8687dfe9175
3
  size 1089994880
train_results.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
  "total_flos": 0.0,
3
- "train_loss": 0.027036830155200475,
4
- "train_runtime": 281082.0852,
5
  "train_samples": 7500,
6
- "train_samples_per_second": 0.027,
7
  "train_steps_per_second": 0.001
8
  }
 
1
  {
2
  "total_flos": 0.0,
3
+ "train_loss": 0.0008256121124141636,
4
+ "train_runtime": 419098.9196,
5
  "train_samples": 7500,
6
+ "train_samples_per_second": 0.018,
7
  "train_steps_per_second": 0.001
8
  }
trainer_state.json CHANGED
@@ -9,1074 +9,1074 @@
9
  "is_world_process_zero": true,
10
  "log_history": [
11
  {
12
- "completion_length": 427.0421132940995,
13
  "epoch": 0.012666666666666666,
14
- "grad_norm": 0.32669389247894287,
15
- "kl": 0.00012453716052205938,
16
  "learning_rate": 3.75e-07,
17
  "loss": 0.0,
18
- "reward": 0.41754386942637595,
19
- "reward_std": 0.3627968850888704,
20
- "rewards/accuracy_reward": 0.41754386942637595,
21
  "rewards/format_reward": 0.0,
22
  "step": 5
23
  },
24
  {
25
- "completion_length": 425.203517552426,
26
  "epoch": 0.025333333333333333,
27
- "grad_norm": 0.17229685187339783,
28
- "kl": 0.00020355676349840667,
29
  "learning_rate": 7.5e-07,
30
  "loss": 0.0,
31
- "reward": 0.44912281726535996,
32
- "reward_std": 0.38365785103095207,
33
- "rewards/accuracy_reward": 0.4473684310913086,
34
- "rewards/format_reward": 0.0017543860171970569,
35
  "step": 10
36
  },
37
  {
38
- "completion_length": 408.717552265368,
39
  "epoch": 0.038,
40
- "grad_norm": 1.1077690124511719,
41
- "kl": 0.000245641407213713,
42
  "learning_rate": 1.125e-06,
43
  "loss": 0.0,
44
- "reward": 0.4894736951903293,
45
- "reward_std": 0.3999988314352537,
46
- "rewards/accuracy_reward": 0.4894736951903293,
47
  "rewards/format_reward": 0.0,
48
  "step": 15
49
  },
50
  {
51
- "completion_length": 427.90176263106497,
52
  "epoch": 0.050666666666666665,
53
- "grad_norm": 0.2539248466491699,
54
- "kl": 0.0002711948595548931,
55
  "learning_rate": 1.5e-06,
56
  "loss": 0.0,
57
- "reward": 0.41403509911737946,
58
- "reward_std": 0.3837312685815912,
59
- "rewards/accuracy_reward": 0.41403509911737946,
60
  "rewards/format_reward": 0.0,
61
  "step": 20
62
  },
63
  {
64
- "completion_length": 433.0245695415296,
65
  "epoch": 0.06333333333333334,
66
- "grad_norm": 0.2533121407032013,
67
- "kl": 0.0006697002210115132,
68
  "learning_rate": 1.875e-06,
69
  "loss": 0.0,
70
- "reward": 0.45087720328255704,
71
- "reward_std": 0.356808750880392,
72
- "rewards/accuracy_reward": 0.45087720328255704,
73
  "rewards/format_reward": 0.0,
74
  "step": 25
75
  },
76
  {
77
- "completion_length": 429.6824665270354,
78
  "epoch": 0.076,
79
- "grad_norm": 0.5356761813163757,
80
- "kl": 0.00229949951171875,
81
  "learning_rate": 2.25e-06,
82
- "loss": 0.0001,
83
- "reward": 0.4789473784597296,
84
- "reward_std": 0.37421235348048965,
85
- "rewards/accuracy_reward": 0.4789473784597296,
86
  "rewards/format_reward": 0.0,
87
  "step": 30
88
  },
89
  {
90
- "completion_length": 457.07193699886926,
91
  "epoch": 0.08866666666666667,
92
- "grad_norm": 0.1357738971710205,
93
- "kl": 0.003329066226356908,
94
  "learning_rate": 2.6250000000000003e-06,
95
- "loss": 0.0001,
96
- "reward": 0.4421052750788237,
97
- "reward_std": 0.3569242841319034,
98
- "rewards/accuracy_reward": 0.4421052750788237,
99
- "rewards/format_reward": 0.0,
100
  "step": 35
101
  },
102
  {
103
- "completion_length": 445.33860441509046,
104
  "epoch": 0.10133333333333333,
105
- "grad_norm": 0.10597766935825348,
106
- "kl": 0.0031187559428967927,
107
  "learning_rate": 3e-06,
108
- "loss": 0.0001,
109
- "reward": 0.45789474844932554,
110
- "reward_std": 0.4124853529428181,
111
- "rewards/accuracy_reward": 0.45789474844932554,
112
  "rewards/format_reward": 0.0,
113
  "step": 40
114
  },
115
  {
116
- "completion_length": 448.7719392475329,
117
  "epoch": 0.114,
118
- "grad_norm": 0.08034415543079376,
119
- "kl": 0.003262630261872944,
120
  "learning_rate": 2.998523534736735e-06,
121
- "loss": 0.0001,
122
- "reward": 0.46666667869216516,
123
- "reward_std": 0.3817528630557813,
124
- "rewards/accuracy_reward": 0.46666667869216516,
125
  "rewards/format_reward": 0.0,
126
  "step": 45
127
  },
128
  {
129
- "completion_length": 443.4122892680921,
130
  "epoch": 0.12666666666666668,
131
- "grad_norm": 0.14021478593349457,
132
- "kl": 0.5420216209010074,
133
  "learning_rate": 2.994097045546504e-06,
134
- "loss": 0.0217,
135
- "reward": 0.48421053682502946,
136
- "reward_std": 0.32790782702596566,
137
- "rewards/accuracy_reward": 0.48421053682502946,
138
  "rewards/format_reward": 0.0,
139
  "step": 50
140
  },
141
  {
142
- "completion_length": 448.9508853310033,
143
  "epoch": 0.13933333333333334,
144
- "grad_norm": 0.6121841073036194,
145
- "kl": 0.006950779965049342,
146
  "learning_rate": 2.986729246506011e-06,
147
- "loss": 0.0003,
148
- "reward": 0.49122808010954605,
149
- "reward_std": 0.3300354468195062,
150
- "rewards/accuracy_reward": 0.49122808010954605,
151
  "rewards/format_reward": 0.0,
152
  "step": 55
153
  },
154
  {
155
- "completion_length": 450.8982540732936,
156
  "epoch": 0.152,
157
- "grad_norm": 0.1517709642648697,
158
- "kl": 0.0043995104337993425,
159
  "learning_rate": 2.976434642014389e-06,
160
- "loss": 0.0002,
161
- "reward": 0.4894736973862899,
162
- "reward_std": 0.37877445064092935,
163
- "rewards/accuracy_reward": 0.4894736973862899,
164
- "rewards/format_reward": 0.0,
165
  "step": 60
166
  },
167
  {
168
- "completion_length": 434.9368501362048,
169
  "epoch": 0.16466666666666666,
170
- "grad_norm": 0.19329309463500977,
171
- "kl": 0.003864890650699013,
172
  "learning_rate": 2.9632334982395456e-06,
173
- "loss": 0.0002,
174
- "reward": 0.5245614123971839,
175
- "reward_std": 0.35019483064350326,
176
- "rewards/accuracy_reward": 0.5245614123971839,
177
  "rewards/format_reward": 0.0,
178
  "step": 65
179
  },
180
  {
181
- "completion_length": 443.1368523848684,
182
  "epoch": 0.17733333333333334,
183
- "grad_norm": 0.005799734033644199,
184
- "kl": 0.0037913272255345395,
185
  "learning_rate": 2.947151803221774e-06,
186
- "loss": 0.0002,
187
- "reward": 0.4964912386317002,
188
- "reward_std": 0.3756070852279663,
189
- "rewards/accuracy_reward": 0.4964912386317002,
190
  "rewards/format_reward": 0.0,
191
  "step": 70
192
  },
193
  {
194
- "completion_length": 451.34737677323193,
195
  "epoch": 0.19,
196
- "grad_norm": 0.12151838839054108,
197
- "kl": 0.004369795949835526,
198
  "learning_rate": 2.928221215713164e-06,
199
- "loss": 0.0002,
200
- "reward": 0.470175449158016,
201
- "reward_std": 0.38960387110710143,
202
- "rewards/accuracy_reward": 0.470175449158016,
203
  "rewards/format_reward": 0.0,
204
  "step": 75
205
  },
206
  {
207
- "completion_length": 434.12632606907897,
208
  "epoch": 0.20266666666666666,
209
- "grad_norm": 0.2288283258676529,
210
- "kl": 0.013469414961965461,
211
  "learning_rate": 2.906479002853542e-06,
212
- "loss": 0.0005,
213
- "reward": 0.48245615080783244,
214
- "reward_std": 0.36099056883862146,
215
- "rewards/accuracy_reward": 0.48245615080783244,
216
  "rewards/format_reward": 0.0,
217
  "step": 80
218
  },
219
  {
220
- "completion_length": 437.27369481136924,
221
  "epoch": 0.21533333333333332,
222
- "grad_norm": 0.18831907212734222,
223
- "kl": 0.015026534231085527,
224
  "learning_rate": 2.8819679668056195e-06,
225
- "loss": 0.0006,
226
- "reward": 0.5157894862325568,
227
- "reward_std": 0.39665700228590717,
228
- "rewards/accuracy_reward": 0.5157894862325568,
229
  "rewards/format_reward": 0.0,
230
  "step": 85
231
  },
232
  {
233
- "completion_length": 440.3596586528577,
234
  "epoch": 0.228,
235
- "grad_norm": 0.004888650495558977,
236
- "kl": 2.206267828690378,
237
  "learning_rate": 2.8547363604937856e-06,
238
- "loss": 0.0884,
239
- "reward": 0.49298246753843206,
240
- "reward_std": 0.3442464342242793,
241
- "rewards/accuracy_reward": 0.49298246753843206,
242
  "rewards/format_reward": 0.0,
243
  "step": 90
244
  },
245
  {
246
- "completion_length": 442.4719376413446,
247
  "epoch": 0.24066666666666667,
248
- "grad_norm": 0.1227022334933281,
249
- "kl": 0.0068352950246710524,
250
  "learning_rate": 2.824837792612416e-06,
251
- "loss": 0.0003,
252
- "reward": 0.5210526447547109,
253
- "reward_std": 0.33440621652101216,
254
- "rewards/accuracy_reward": 0.5210526447547109,
255
  "rewards/format_reward": 0.0,
256
  "step": 95
257
  },
258
  {
259
- "completion_length": 431.6807120875308,
260
  "epoch": 0.25333333333333335,
261
- "grad_norm": 0.18928714096546173,
262
- "kl": 0.008197182103207237,
263
  "learning_rate": 2.792331122090709e-06,
264
- "loss": 0.0003,
265
- "reward": 0.5385965028875752,
266
- "reward_std": 0.35269192080748707,
267
- "rewards/accuracy_reward": 0.5385965028875752,
268
  "rewards/format_reward": 0.0,
269
  "step": 100
270
  },
271
  {
272
  "epoch": 0.25333333333333335,
273
- "eval_completion_length": 435.9969423065186,
274
- "eval_kl": 0.009833270263671876,
275
- "eval_loss": 0.00039347587153315544,
276
- "eval_reward": 0.4568000102430582,
277
- "eval_reward_std": 0.3654944499373436,
278
- "eval_rewards/accuracy_reward": 0.4568000102430582,
279
  "eval_rewards/format_reward": 0.0,
280
- "eval_runtime": 48530.4928,
281
- "eval_samples_per_second": 0.103,
282
- "eval_steps_per_second": 0.017,
283
  "step": 100
284
  },
285
  {
286
- "completion_length": 437.9824650814659,
287
  "epoch": 0.266,
288
- "grad_norm": 0.12078626453876495,
289
- "kl": 0.008320858604029605,
290
  "learning_rate": 2.7572803422217976e-06,
291
- "loss": 0.0003,
292
- "reward": 0.5263158009240502,
293
- "reward_std": 0.37190421505978233,
294
- "rewards/accuracy_reward": 0.5263158009240502,
295
  "rewards/format_reward": 0.0,
296
  "step": 105
297
  },
298
  {
299
- "completion_length": 439.51228991056746,
300
  "epoch": 0.2786666666666667,
301
- "grad_norm": 0.13941198587417603,
302
- "kl": 0.0077880859375,
303
  "learning_rate": 2.71975445468425e-06,
304
- "loss": 0.0003,
305
- "reward": 0.49122808167808935,
306
- "reward_std": 0.4313258014227215,
307
- "rewards/accuracy_reward": 0.49122808167808935,
308
  "rewards/format_reward": 0.0,
309
  "step": 110
310
  },
311
  {
312
- "completion_length": 421.4017642372533,
313
  "epoch": 0.29133333333333333,
314
- "grad_norm": 0.10381676256656647,
315
- "kl": 0.00989588687294408,
316
  "learning_rate": 2.679827333703964e-06,
317
- "loss": 0.0004,
318
- "reward": 0.5719298359594847,
319
- "reward_std": 0.3785097053176478,
320
- "rewards/accuracy_reward": 0.5719298359594847,
321
  "rewards/format_reward": 0.0,
322
  "step": 115
323
  },
324
  {
325
- "completion_length": 421.6526407342208,
326
  "epoch": 0.304,
327
- "grad_norm": 0.2092106193304062,
328
- "kl": 0.010035464638157895,
329
  "learning_rate": 2.637577580623858e-06,
330
- "loss": 0.0004,
331
- "reward": 0.5859649266067304,
332
- "reward_std": 0.3874762525683955,
333
- "rewards/accuracy_reward": 0.5859649266067304,
334
  "rewards/format_reward": 0.0,
335
  "step": 120
336
  },
337
  {
338
- "completion_length": 414.329833984375,
339
  "epoch": 0.31666666666666665,
340
- "grad_norm": 0.23555059731006622,
341
- "kl": 0.009284732216282895,
342
  "learning_rate": 2.593088369167671e-06,
343
- "loss": 0.0004,
344
- "reward": 0.62280702794853,
345
- "reward_std": 0.32933386564254763,
346
- "rewards/accuracy_reward": 0.62280702794853,
347
  "rewards/format_reward": 0.0,
348
  "step": 125
349
  },
350
  {
351
- "completion_length": 417.16316656815377,
352
  "epoch": 0.3293333333333333,
353
- "grad_norm": 0.12066253274679184,
354
- "kl": 0.012572599712171053,
355
  "learning_rate": 2.5464472817024772e-06,
356
- "loss": 0.0005,
357
- "reward": 0.6175438719360452,
358
- "reward_std": 0.3299596526120838,
359
- "rewards/accuracy_reward": 0.6175438719360452,
360
  "rewards/format_reward": 0.0,
361
  "step": 130
362
  },
363
  {
364
- "completion_length": 429.32106323242186,
365
  "epoch": 0.342,
366
- "grad_norm": 0.14316421747207642,
367
- "kl": 0.015025570518092106,
368
  "learning_rate": 2.497746136822254e-06,
369
- "loss": 0.0006,
370
- "reward": 0.5842105401189703,
371
- "reward_std": 0.3870730996131897,
372
- "rewards/accuracy_reward": 0.5842105401189703,
373
  "rewards/format_reward": 0.0,
374
  "step": 135
375
  },
376
  {
377
- "completion_length": 429.531587942023,
378
  "epoch": 0.3546666666666667,
379
- "grad_norm": 0.14851997792720795,
380
- "kl": 0.02211560701069079,
381
  "learning_rate": 2.4470808085919304e-06,
382
- "loss": 0.0009,
383
- "reward": 0.5649122933023855,
384
- "reward_std": 0.35544770171767787,
385
- "rewards/accuracy_reward": 0.5649122933023855,
386
  "rewards/format_reward": 0.0,
387
  "step": 140
388
  },
389
  {
390
- "completion_length": 447.08246427837173,
391
  "epoch": 0.36733333333333335,
392
- "grad_norm": 0.10792040824890137,
393
- "kl": 0.017962325246710525,
394
  "learning_rate": 2.3945510378077523e-06,
395
- "loss": 0.0007,
396
- "reward": 0.5122807138844541,
397
- "reward_std": 0.35567033479088234,
398
- "rewards/accuracy_reward": 0.5122807138844541,
399
  "rewards/format_reward": 0.0,
400
  "step": 145
401
  },
402
  {
403
- "completion_length": 423.1122905530428,
404
  "epoch": 0.38,
405
- "grad_norm": 0.11955911666154861,
406
- "kl": 0.02356021278782895,
407
  "learning_rate": 2.340260235645519e-06,
408
- "loss": 0.0009,
409
- "reward": 0.5578947500178688,
410
- "reward_std": 0.36022400009004696,
411
- "rewards/accuracy_reward": 0.5578947500178688,
412
  "rewards/format_reward": 0.0,
413
  "step": 150
414
  },
415
  {
416
- "completion_length": 409.61053514982524,
417
  "epoch": 0.39266666666666666,
418
- "grad_norm": 0.09739726036787033,
419
- "kl": 0.018040385999177632,
420
  "learning_rate": 2.2843152800832416e-06,
421
- "loss": 0.0007,
422
- "reward": 0.6157894856051395,
423
- "reward_std": 0.34050145086489225,
424
- "rewards/accuracy_reward": 0.6157894856051395,
425
  "rewards/format_reward": 0.0,
426
  "step": 155
427
  },
428
  {
429
- "completion_length": 404.34913137335525,
430
  "epoch": 0.4053333333333333,
431
- "grad_norm": 0.004406505264341831,
432
- "kl": 0.02299708316200658,
433
  "learning_rate": 2.2268263054989753e-06,
434
- "loss": 0.0009,
435
- "reward": 0.5842105368250294,
436
- "reward_std": 0.3406506648189143,
437
- "rewards/accuracy_reward": 0.5842105368250294,
438
  "rewards/format_reward": 0.0,
439
  "step": 160
440
  },
441
  {
442
- "completion_length": 407.5386057000411,
443
  "epoch": 0.418,
444
- "grad_norm": 0.18651004135608673,
445
- "kl": 0.05064440275493421,
446
  "learning_rate": 2.167906485858047e-06,
447
- "loss": 0.002,
448
- "reward": 0.6210526471075258,
449
- "reward_std": 0.32974544889048524,
450
- "rewards/accuracy_reward": 0.6210526471075258,
451
  "rewards/format_reward": 0.0,
452
  "step": 165
453
  },
454
  {
455
- "completion_length": 409.0772017629523,
456
  "epoch": 0.43066666666666664,
457
- "grad_norm": 0.13046123087406158,
458
- "kl": 0.06625655324835526,
459
  "learning_rate": 2.1076718119164804e-06,
460
- "loss": 0.0026,
461
- "reward": 0.5631579061872081,
462
- "reward_std": 0.37127605362942345,
463
- "rewards/accuracy_reward": 0.5631579061872081,
464
  "rewards/format_reward": 0.0,
465
  "step": 170
466
  },
467
  {
468
- "completion_length": 402.12983334189965,
469
  "epoch": 0.44333333333333336,
470
- "grad_norm": 0.10294575989246368,
471
- "kl": 0.027840383429276316,
472
  "learning_rate": 2.0462408628792335e-06,
473
- "loss": 0.0011,
474
- "reward": 0.6105263289652373,
475
- "reward_std": 0.35427797938648026,
476
- "rewards/accuracy_reward": 0.6105263289652373,
477
  "rewards/format_reward": 0.0,
478
  "step": 175
479
  },
480
  {
481
- "completion_length": 419.62632414165296,
482
  "epoch": 0.456,
483
- "grad_norm": 0.20833256840705872,
484
- "kl": 0.029789974814967105,
485
  "learning_rate": 1.9837345729627633e-06,
486
- "loss": 0.0012,
487
- "reward": 0.5614035218954087,
488
- "reward_std": 0.3542021836105146,
489
- "rewards/accuracy_reward": 0.5614035218954087,
490
  "rewards/format_reward": 0.0,
491
  "step": 180
492
  },
493
  {
494
- "completion_length": 417.81930590177836,
495
  "epoch": 0.4686666666666667,
496
- "grad_norm": 0.012007645331323147,
497
- "kl": 0.02894447728207237,
498
  "learning_rate": 1.9202759933214665e-06,
499
- "loss": 0.0012,
500
- "reward": 0.5105263262987136,
501
- "reward_std": 0.34277828335762023,
502
- "rewards/accuracy_reward": 0.5105263262987136,
503
  "rewards/format_reward": 0.0,
504
  "step": 185
505
  },
506
  {
507
- "completion_length": 395.12457034462375,
508
  "epoch": 0.48133333333333334,
509
- "grad_norm": 0.15910613536834717,
510
- "kl": 0.031014532791940788,
511
  "learning_rate": 1.8559900498066726e-06,
512
- "loss": 0.0012,
513
- "reward": 0.5754386062684812,
514
- "reward_std": 0.31805680268689207,
515
- "rewards/accuracy_reward": 0.5754386062684812,
516
  "rewards/format_reward": 0.0,
517
  "step": 190
518
  },
519
  {
520
- "completion_length": 399.89123856393917,
521
  "epoch": 0.494,
522
- "grad_norm": 0.20715074241161346,
523
- "kl": 0.03553081311677632,
524
  "learning_rate": 1.7910032970350677e-06,
525
- "loss": 0.0014,
526
- "reward": 0.5456140458583831,
527
- "reward_std": 0.37994654994261895,
528
- "rewards/accuracy_reward": 0.5456140458583831,
529
  "rewards/format_reward": 0.0,
530
  "step": 195
531
  },
532
  {
533
- "completion_length": 407.15264089483964,
534
  "epoch": 0.5066666666666667,
535
- "grad_norm": 0.29204627871513367,
536
- "kl": 0.03386648077713816,
537
  "learning_rate": 1.7254436692507058e-06,
538
- "loss": 0.0014,
539
- "reward": 0.5596491352507943,
540
- "reward_std": 0.3601168986998106,
541
- "rewards/accuracy_reward": 0.5596491352507943,
542
  "rewards/format_reward": 0.0,
543
  "step": 200
544
  },
545
  {
546
  "epoch": 0.5066666666666667,
547
- "eval_completion_length": 394.2290429840088,
548
- "eval_kl": 0.04040946655273438,
549
- "eval_loss": 0.0015893690288066864,
550
- "eval_reward": 0.49966667771935463,
551
- "eval_reward_std": 0.3627293815135956,
552
- "eval_rewards/accuracy_reward": 0.49966667771935463,
553
  "eval_rewards/format_reward": 0.0,
554
- "eval_runtime": 47257.0811,
555
- "eval_samples_per_second": 0.106,
556
- "eval_steps_per_second": 0.018,
557
  "step": 200
558
  },
559
  {
560
- "completion_length": 397.5421145790502,
561
  "epoch": 0.5193333333333333,
562
- "grad_norm": 0.1434248983860016,
563
- "kl": 0.046006373355263155,
564
  "learning_rate": 1.6594402284710481e-06,
565
- "loss": 0.0018,
566
- "reward": 0.6000000109798029,
567
- "reward_std": 0.3170362949371338,
568
- "rewards/accuracy_reward": 0.6000000109798029,
569
  "rewards/format_reward": 0.0,
570
  "step": 205
571
  },
572
  {
573
- "completion_length": 389.34562474300986,
574
  "epoch": 0.532,
575
- "grad_norm": 0.21728208661079407,
576
- "kl": 0.06839278371710526,
577
  "learning_rate": 1.593122910412851e-06,
578
- "loss": 0.0027,
579
- "reward": 0.591228081991798,
580
- "reward_std": 0.3792112864946064,
581
- "rewards/accuracy_reward": 0.591228081991798,
582
  "rewards/format_reward": 0.0,
583
  "step": 210
584
  },
585
  {
586
- "completion_length": 393.13334222090873,
587
  "epoch": 0.5446666666666666,
588
- "grad_norm": 4.170770168304443,
589
- "kl": 0.08757773951480263,
590
  "learning_rate": 1.5266222686980693e-06,
591
- "loss": 0.0035,
592
- "reward": 0.41929825356132105,
593
- "reward_std": 0.33843882052521956,
594
- "rewards/accuracy_reward": 0.41929825356132105,
595
  "rewards/format_reward": 0.0,
596
  "step": 215
597
  },
598
  {
599
- "completion_length": 402.3175527472245,
600
  "epoch": 0.5573333333333333,
601
- "grad_norm": 0.04682053253054619,
602
- "kl": 0.10470291940789474,
603
  "learning_rate": 1.460069217843338e-06,
604
- "loss": 0.0042,
605
- "reward": 0.3122807102768045,
606
- "reward_std": 0.35611560125099984,
607
- "rewards/accuracy_reward": 0.3122807102768045,
608
  "rewards/format_reward": 0.0,
609
  "step": 220
610
  },
611
  {
612
- "completion_length": 421.3649226138466,
613
  "epoch": 0.57,
614
- "grad_norm": 0.3474079668521881,
615
- "kl": 0.09076377467105264,
616
  "learning_rate": 1.3935947755389924e-06,
617
- "loss": 0.0036,
618
- "reward": 0.27719298852117435,
619
- "reward_std": 0.33377805352211,
620
- "rewards/accuracy_reward": 0.27719298852117435,
621
  "rewards/format_reward": 0.0,
622
  "step": 225
623
  },
624
  {
625
- "completion_length": 413.22983430561266,
626
  "epoch": 0.5826666666666667,
627
- "grad_norm": 0.5406464338302612,
628
- "kl": 0.07656506990131579,
629
  "learning_rate": 1.3273298047249756e-06,
630
- "loss": 0.0031,
631
- "reward": 0.3421052706869025,
632
- "reward_std": 0.40586905918623273,
633
- "rewards/accuracy_reward": 0.3421052706869025,
634
  "rewards/format_reward": 0.0,
635
  "step": 230
636
  },
637
  {
638
- "completion_length": 391.9754479659231,
639
  "epoch": 0.5953333333333334,
640
- "grad_norm": 0.6628542542457581,
641
- "kl": 0.07627788342927631,
642
  "learning_rate": 1.2614047559713923e-06,
643
- "loss": 0.0031,
644
- "reward": 0.47192983580263037,
645
- "reward_std": 0.3767852387930218,
646
- "rewards/accuracy_reward": 0.47192983580263037,
647
  "rewards/format_reward": 0.0,
648
  "step": 235
649
  },
650
  {
651
- "completion_length": 398.3614137348376,
652
  "epoch": 0.608,
653
- "grad_norm": 0.22894343733787537,
654
- "kl": 0.09133493523848685,
655
  "learning_rate": 1.1959494106708598e-06,
656
- "loss": 0.0037,
657
- "reward": 0.4543859756306598,
658
- "reward_std": 0.41446375846862793,
659
- "rewards/accuracy_reward": 0.4543859756306598,
660
  "rewards/format_reward": 0.0,
661
  "step": 240
662
  },
663
  {
664
- "completion_length": 382.73158842387954,
665
  "epoch": 0.6206666666666667,
666
- "grad_norm": 0.2360672801733017,
667
- "kl": 0.12881951583059212,
668
  "learning_rate": 1.1310926255482204e-06,
669
- "loss": 0.0052,
670
- "reward": 0.5070175553622999,
671
- "reward_std": 0.38596598882424205,
672
- "rewards/accuracy_reward": 0.5070175553622999,
673
  "rewards/format_reward": 0.0,
674
  "step": 245
675
  },
676
  {
677
- "completion_length": 379.06843085038037,
678
  "epoch": 0.6333333333333333,
679
- "grad_norm": 0.41622936725616455,
680
- "kl": 0.4508557771381579,
681
  "learning_rate": 1.0669620789905688e-06,
682
- "loss": 0.018,
683
- "reward": 0.5122807151392886,
684
- "reward_std": 0.35713848534383275,
685
- "rewards/accuracy_reward": 0.5122807151392886,
686
  "rewards/format_reward": 0.0,
687
  "step": 250
688
  },
689
  {
690
- "completion_length": 379.2842171116879,
691
  "epoch": 0.646,
692
- "grad_norm": 2.963742256164551,
693
- "kl": 1.111646792763158,
694
  "learning_rate": 1.0036840196969795e-06,
695
- "loss": 0.0445,
696
- "reward": 0.5614035189151764,
697
- "reward_std": 0.33363121528374523,
698
- "rewards/accuracy_reward": 0.5614035189151764,
699
  "rewards/format_reward": 0.0,
700
  "step": 255
701
  },
702
  {
703
- "completion_length": 390.0175530684622,
704
  "epoch": 0.6586666666666666,
705
- "grad_norm": 1.9537420272827148,
706
- "kl": 1.1658845600328946,
707
  "learning_rate": 9.413830181427508e-07,
708
- "loss": 0.0466,
709
- "reward": 0.5263158009240502,
710
- "reward_std": 0.3237152021182211,
711
- "rewards/accuracy_reward": 0.5263158009240502,
712
  "rewards/format_reward": 0.0,
713
  "step": 260
714
  },
715
  {
716
- "completion_length": 406.68421936035156,
717
  "epoch": 0.6713333333333333,
718
- "grad_norm": 0.1643250286579132,
719
- "kl": 9.794161184210527,
720
  "learning_rate": 8.801817213474331e-07,
721
- "loss": 0.3924,
722
- "reward": 0.44210527413769773,
723
- "reward_std": 0.3688186996861508,
724
- "rewards/accuracy_reward": 0.44210527413769773,
725
  "rewards/format_reward": 0.0,
726
  "step": 265
727
  },
728
  {
729
- "completion_length": 394.60878087093954,
730
  "epoch": 0.684,
731
- "grad_norm": 0.5662396550178528,
732
- "kl": 0.29067961040296053,
733
  "learning_rate": 8.202006114294044e-07,
734
- "loss": 0.0116,
735
- "reward": 0.47368422009442984,
736
- "reward_std": 0.37373340569044416,
737
- "rewards/accuracy_reward": 0.47368422009442984,
738
  "rewards/format_reward": 0.0,
739
  "step": 270
740
  },
741
  {
742
- "completion_length": 383.15790437397203,
743
  "epoch": 0.6966666666666667,
744
- "grad_norm": 0.36152008175849915,
745
- "kl": 0.8754098992598685,
746
  "learning_rate": 7.615577684223272e-07,
747
- "loss": 0.035,
748
- "reward": 0.5210526431861676,
749
- "reward_std": 0.39000940040538185,
750
- "rewards/accuracy_reward": 0.5210526431861676,
751
  "rewards/format_reward": 0.0,
752
  "step": 275
753
  },
754
  {
755
- "completion_length": 396.80176439787215,
756
  "epoch": 0.7093333333333334,
757
- "grad_norm": 1.2584236860275269,
758
- "kl": 1.3499331825657894,
759
  "learning_rate": 7.043686378203864e-07,
760
- "loss": 0.054,
761
- "reward": 0.4596491366624832,
762
- "reward_std": 0.37697656687937287,
763
- "rewards/accuracy_reward": 0.4596491366624832,
764
  "rewards/format_reward": 0.0,
765
  "step": 280
766
  },
767
  {
768
- "completion_length": 408.31755178351153,
769
  "epoch": 0.722,
770
- "grad_norm": 1.6444029808044434,
771
- "kl": 0.37659269634046055,
772
  "learning_rate": 6.487458033099425e-07,
773
- "loss": 0.0151,
774
- "reward": 0.43333334467912976,
775
- "reward_std": 0.41270798601602254,
776
- "rewards/accuracy_reward": 0.43333334467912976,
777
  "rewards/format_reward": 0.0,
778
  "step": 285
779
  },
780
  {
781
- "completion_length": 396.7526412160773,
782
  "epoch": 0.7346666666666667,
783
- "grad_norm": 0.24537613987922668,
784
- "kl": 0.7064022666529606,
785
  "learning_rate": 5.947987651349942e-07,
786
- "loss": 0.0283,
787
- "reward": 0.5245614156911248,
788
- "reward_std": 0.39676410242130883,
789
- "rewards/accuracy_reward": 0.5245614156911248,
790
  "rewards/format_reward": 0.0,
791
  "step": 290
792
  },
793
  {
794
- "completion_length": 396.04386933979237,
795
  "epoch": 0.7473333333333333,
796
- "grad_norm": 1.1248526573181152,
797
- "kl": 2.2961811266447367,
798
  "learning_rate": 5.426337245327703e-07,
799
- "loss": 0.0919,
800
- "reward": 0.5070175574014061,
801
- "reward_std": 0.3869973038372241,
802
- "rewards/accuracy_reward": 0.5070175574014061,
803
  "rewards/format_reward": 0.0,
804
  "step": 295
805
  },
806
  {
807
- "completion_length": 393.7561492919922,
808
  "epoch": 0.76,
809
- "grad_norm": 5.84006929397583,
810
- "kl": 5.933592465049342,
811
  "learning_rate": 4.923533746638108e-07,
812
- "loss": 0.2375,
813
- "reward": 0.48245614986670643,
814
- "reward_std": 0.40506037599162054,
815
- "rewards/accuracy_reward": 0.48245614986670643,
816
  "rewards/format_reward": 0.0,
817
  "step": 300
818
  },
819
  {
820
  "epoch": 0.76,
821
- "eval_completion_length": 393.75374295043946,
822
- "eval_kl": 3.580558123779297,
823
- "eval_loss": 0.14329032599925995,
824
- "eval_reward": 0.41980000983178617,
825
- "eval_reward_std": 0.3639850652396679,
826
- "eval_rewards/accuracy_reward": 0.41980000983178617,
827
  "eval_rewards/format_reward": 0.0,
828
- "eval_runtime": 47686.4832,
829
- "eval_samples_per_second": 0.105,
830
- "eval_steps_per_second": 0.017,
831
  "step": 300
832
  },
833
  {
834
- "completion_length": 406.72983366313736,
835
  "epoch": 0.7726666666666666,
836
- "grad_norm": 0.570837140083313,
837
- "kl": 2.009347373560855,
838
  "learning_rate": 4.440566984481256e-07,
839
- "loss": 0.0804,
840
- "reward": 0.42280702669369546,
841
- "reward_std": 0.4078474631434993,
842
- "rewards/accuracy_reward": 0.42280702669369546,
843
  "rewards/format_reward": 0.0,
844
  "step": 305
845
  },
846
  {
847
- "completion_length": 387.854396458676,
848
  "epoch": 0.7853333333333333,
849
- "grad_norm": 0.4837806224822998,
850
- "kl": 0.8569181743421053,
851
  "learning_rate": 3.978387737053994e-07,
852
- "loss": 0.0343,
853
- "reward": 0.5350877289709292,
854
- "reward_std": 0.3191217996572193,
855
- "rewards/accuracy_reward": 0.5350877289709292,
856
  "rewards/format_reward": 0.0,
857
  "step": 310
858
  },
859
  {
860
- "completion_length": 391.2175537109375,
861
  "epoch": 0.798,
862
- "grad_norm": 1.2316937446594238,
863
- "kl": 1.243909333881579,
864
  "learning_rate": 3.5379058598286167e-07,
865
- "loss": 0.0497,
866
- "reward": 0.4842105376093011,
867
- "reward_std": 0.3719776351200907,
868
- "rewards/accuracy_reward": 0.4842105376093011,
869
  "rewards/format_reward": 0.0,
870
  "step": 315
871
  },
872
  {
873
- "completion_length": 404.43509698165093,
874
  "epoch": 0.8106666666666666,
875
- "grad_norm": 1.2691428661346436,
876
- "kl": 1.967847964638158,
877
  "learning_rate": 3.119988494392894e-07,
878
- "loss": 0.0787,
879
- "reward": 0.47543860705275287,
880
- "reward_std": 0.40146460689996416,
881
- "rewards/accuracy_reward": 0.47543860705275287,
882
  "rewards/format_reward": 0.0,
883
  "step": 320
884
  },
885
  {
886
- "completion_length": 385.6350979453639,
887
  "epoch": 0.8233333333333334,
888
- "grad_norm": 2.1663596630096436,
889
- "kl": 1.7478567023026317,
890
  "learning_rate": 2.725458361377465e-07,
891
- "loss": 0.0699,
892
- "reward": 0.5333333443654211,
893
- "reward_std": 0.3384051385678743,
894
- "rewards/accuracy_reward": 0.5333333443654211,
895
  "rewards/format_reward": 0.0,
896
  "step": 325
897
  },
898
  {
899
- "completion_length": 389.51755499588813,
900
  "epoch": 0.836,
901
- "grad_norm": 0.9927846789360046,
902
- "kl": 2.0717015316611844,
903
  "learning_rate": 2.3550921408312737e-07,
904
- "loss": 0.0829,
905
- "reward": 0.5052631704430831,
906
- "reward_std": 0.386816783641514,
907
- "rewards/accuracy_reward": 0.5052631704430831,
908
  "rewards/format_reward": 0.0,
909
  "step": 330
910
  },
911
  {
912
- "completion_length": 380.80000770970395,
913
  "epoch": 0.8486666666666667,
914
- "grad_norm": 1.9131776094436646,
915
- "kl": 1.4728098016036184,
916
  "learning_rate": 2.0096189432334195e-07,
917
- "loss": 0.059,
918
- "reward": 0.46491229016529884,
919
- "reward_std": 0.3830212574256094,
920
- "rewards/accuracy_reward": 0.46491229016529884,
921
  "rewards/format_reward": 0.0,
922
  "step": 335
923
  },
924
  {
925
- "completion_length": 384.0947445518092,
926
  "epoch": 0.8613333333333333,
927
- "grad_norm": 0.5461324453353882,
928
- "kl": 1.2412668328536185,
929
  "learning_rate": 1.6897188741514286e-07,
930
- "loss": 0.0497,
931
- "reward": 0.46842106282711027,
932
- "reward_std": 0.372008940106944,
933
- "rewards/accuracy_reward": 0.46842106282711027,
934
  "rewards/format_reward": 0.0,
935
  "step": 340
936
  },
937
  {
938
- "completion_length": 395.47720272666527,
939
  "epoch": 0.874,
940
- "grad_norm": 0.5957249999046326,
941
- "kl": 1.3121890419407896,
942
  "learning_rate": 1.396021695371582e-07,
943
- "loss": 0.0525,
944
- "reward": 0.4543859761012228,
945
- "reward_std": 0.35364981732870404,
946
- "rewards/accuracy_reward": 0.4543859761012228,
947
  "rewards/format_reward": 0.0,
948
  "step": 345
949
  },
950
  {
951
- "completion_length": 388.5894830001028,
952
  "epoch": 0.8866666666666667,
953
- "grad_norm": 0.7833815813064575,
954
- "kl": 1.2672890111019737,
955
  "learning_rate": 1.1291055851370623e-07,
956
- "loss": 0.0507,
957
- "reward": 0.46491229063586187,
958
- "reward_std": 0.3626813524647763,
959
- "rewards/accuracy_reward": 0.46491229063586187,
960
  "rewards/format_reward": 0.0,
961
  "step": 350
962
  },
963
  {
964
- "completion_length": 393.6350974635074,
965
  "epoch": 0.8993333333333333,
966
- "grad_norm": 1.0711956024169922,
967
- "kl": 1.032947419819079,
968
  "learning_rate": 8.894959999345015e-08,
969
- "loss": 0.0413,
970
- "reward": 0.4210526403627898,
971
- "reward_std": 0.3503103620127628,
972
- "rewards/accuracy_reward": 0.4210526403627898,
973
  "rewards/format_reward": 0.0,
974
  "step": 355
975
  },
976
  {
977
- "completion_length": 389.8333436664782,
978
  "epoch": 0.912,
979
- "grad_norm": 0.6254836320877075,
980
- "kl": 0.9737092670641447,
981
  "learning_rate": 6.776646400696212e-08,
982
- "loss": 0.039,
983
- "reward": 0.4859649231559352,
984
- "reward_std": 0.38237022004629434,
985
- "rewards/accuracy_reward": 0.4859649231559352,
986
  "rewards/format_reward": 0.0,
987
  "step": 360
988
  },
989
  {
990
- "completion_length": 389.48948348195927,
991
  "epoch": 0.9246666666666666,
992
- "grad_norm": 1.6106293201446533,
993
- "kl": 0.9006559673108553,
994
  "learning_rate": 4.940285210684375e-08,
995
- "loss": 0.036,
996
- "reward": 0.49122808324663264,
997
- "reward_std": 0.3813075975367897,
998
- "rewards/accuracy_reward": 0.49122808324663264,
999
  "rewards/format_reward": 0.0,
1000
  "step": 365
1001
  },
1002
  {
1003
- "completion_length": 387.6245711477179,
1004
  "epoch": 0.9373333333333334,
1005
- "grad_norm": 2.270439863204956,
1006
- "kl": 1.5312891909950659,
1007
  "learning_rate": 3.389491527319999e-08,
1008
- "loss": 0.0613,
1009
- "reward": 0.4543859757875141,
1010
- "reward_std": 0.3992635683009499,
1011
- "rewards/accuracy_reward": 0.4543859757875141,
1012
  "rewards/format_reward": 0.0,
1013
  "step": 370
1014
  },
1015
  {
1016
- "completion_length": 398.4315892269737,
1017
  "epoch": 0.95,
1018
- "grad_norm": 2.971569061279297,
1019
- "kl": 1.466099146792763,
1020
  "learning_rate": 2.127318274608381e-08,
1021
- "loss": 0.0586,
1022
- "reward": 0.4473684310913086,
1023
- "reward_std": 0.30884474672769247,
1024
- "rewards/accuracy_reward": 0.4473684310913086,
1025
  "rewards/format_reward": 0.0,
1026
  "step": 375
1027
  },
1028
  {
1029
- "completion_length": 382.3105351498252,
1030
  "epoch": 0.9626666666666667,
1031
- "grad_norm": 0.4336649775505066,
1032
- "kl": 1.0901534231085526,
1033
  "learning_rate": 1.1562501925013125e-08,
1034
- "loss": 0.0436,
1035
- "reward": 0.5385965030444296,
1036
- "reward_std": 0.3790620725405844,
1037
- "rewards/accuracy_reward": 0.5385965030444296,
1038
  "rewards/format_reward": 0.0,
1039
  "step": 380
1040
  },
1041
  {
1042
- "completion_length": 382.67018448679073,
1043
  "epoch": 0.9753333333333334,
1044
- "grad_norm": 2.1975314617156982,
1045
- "kl": 1.1284366005345394,
1046
  "learning_rate": 4.781989453874814e-09,
1047
- "loss": 0.0452,
1048
- "reward": 0.5087719407520797,
1049
- "reward_std": 0.3483945708525808,
1050
- "rewards/accuracy_reward": 0.5087719407520797,
1051
  "rewards/format_reward": 0.0,
1052
  "step": 385
1053
  },
1054
  {
1055
- "completion_length": 395.0982547157689,
1056
  "epoch": 0.988,
1057
- "grad_norm": 1.3517661094665527,
1058
- "kl": 1.2858867444490132,
1059
  "learning_rate": 9.44993587509657e-10,
1060
- "loss": 0.0514,
1061
- "reward": 0.5070175545780282,
1062
- "reward_std": 0.4019014430673499,
1063
- "rewards/accuracy_reward": 0.5070175545780282,
1064
  "rewards/format_reward": 0.0,
1065
  "step": 390
1066
  },
1067
  {
1068
- "completion_length": 403.1096580906918,
1069
  "epoch": 0.9981333333333333,
1070
- "kl": 2.4145796926398027,
1071
- "reward": 0.42324562468811083,
1072
- "reward_std": 0.32168745288723394,
1073
- "rewards/accuracy_reward": 0.42324562468811083,
1074
  "rewards/format_reward": 0.0,
1075
  "step": 394,
1076
  "total_flos": 0.0,
1077
- "train_loss": 0.027036830155200475,
1078
- "train_runtime": 281082.0852,
1079
- "train_samples_per_second": 0.027,
1080
  "train_steps_per_second": 0.001
1081
  }
1082
  ],
 
9
  "is_world_process_zero": true,
10
  "log_history": [
11
  {
12
+ "completion_length": 644.0193140933388,
13
  "epoch": 0.012666666666666666,
14
+ "grad_norm": 0.16860076785087585,
15
+ "kl": 0.000124662016567431,
16
  "learning_rate": 3.75e-07,
17
  "loss": 0.0,
18
+ "reward": 0.605263171384209,
19
+ "reward_std": 0.40058013012534693,
20
+ "rewards/accuracy_reward": 0.605263171384209,
21
  "rewards/format_reward": 0.0,
22
  "step": 5
23
  },
24
  {
25
+ "completion_length": 601.6614225688734,
26
  "epoch": 0.025333333333333333,
27
+ "grad_norm": 0.14201320707798004,
28
+ "kl": 0.00020345888639751233,
29
  "learning_rate": 7.5e-07,
30
  "loss": 0.0,
31
+ "reward": 0.673684226839166,
32
+ "reward_std": 0.39511305846666034,
33
+ "rewards/accuracy_reward": 0.673684226839166,
34
+ "rewards/format_reward": 0.0,
35
  "step": 10
36
  },
37
  {
38
+ "completion_length": 579.6684371145149,
39
  "epoch": 0.038,
40
+ "grad_norm": 0.3944288194179535,
41
+ "kl": 0.00023700312564247533,
42
  "learning_rate": 1.125e-06,
43
  "loss": 0.0,
44
+ "reward": 0.6631579123045269,
45
+ "reward_std": 0.3630171424464176,
46
+ "rewards/accuracy_reward": 0.6631579123045269,
47
  "rewards/format_reward": 0.0,
48
  "step": 15
49
  },
50
  {
51
+ "completion_length": 615.7789647152549,
52
  "epoch": 0.050666666666666665,
53
+ "grad_norm": 0.1742940992116928,
54
+ "kl": 0.0005895815397563733,
55
  "learning_rate": 1.5e-06,
56
  "loss": 0.0,
57
+ "reward": 0.6175438761711121,
58
+ "reward_std": 0.40319750308990476,
59
+ "rewards/accuracy_reward": 0.6175438761711121,
60
  "rewards/format_reward": 0.0,
61
  "step": 20
62
  },
63
  {
64
+ "completion_length": 653.6140499717311,
65
  "epoch": 0.06333333333333334,
66
+ "grad_norm": 0.213926300406456,
67
+ "kl": 0.0014986289174933182,
68
  "learning_rate": 1.875e-06,
69
  "loss": 0.0,
70
+ "reward": 0.6561403632164001,
71
+ "reward_std": 0.3638197688679946,
72
+ "rewards/accuracy_reward": 0.6561403632164001,
73
  "rewards/format_reward": 0.0,
74
  "step": 25
75
  },
76
  {
77
+ "completion_length": 624.1438784950658,
78
  "epoch": 0.076,
79
+ "grad_norm": 0.16140718758106232,
80
+ "kl": 0.0023109034488075658,
81
  "learning_rate": 2.25e-06,
82
+ "loss": 0.0,
83
+ "reward": 0.6947368581044047,
84
+ "reward_std": 0.32959623650500647,
85
+ "rewards/accuracy_reward": 0.6947368581044047,
86
  "rewards/format_reward": 0.0,
87
  "step": 30
88
  },
89
  {
90
+ "completion_length": 649.8438781738281,
91
  "epoch": 0.08866666666666667,
92
+ "grad_norm": 0.16571274399757385,
93
+ "kl": 0.009154390033922698,
94
  "learning_rate": 2.6250000000000003e-06,
95
+ "loss": 0.0,
96
+ "reward": 0.6894737009939395,
97
+ "reward_std": 0.35860375699244046,
98
+ "rewards/accuracy_reward": 0.687719314819888,
99
+ "rewards/format_reward": 0.0017543860171970569,
100
  "step": 35
101
  },
102
  {
103
+ "completion_length": 629.4052797016344,
104
  "epoch": 0.10133333333333333,
105
+ "grad_norm": 0.27635857462882996,
106
+ "kl": 0.0027755938078227794,
107
  "learning_rate": 3e-06,
108
+ "loss": 0.0,
109
+ "reward": 0.7035087861512836,
110
+ "reward_std": 0.34189380752412896,
111
+ "rewards/accuracy_reward": 0.7035087861512836,
112
  "rewards/format_reward": 0.0,
113
  "step": 40
114
  },
115
  {
116
+ "completion_length": 652.133349609375,
117
  "epoch": 0.114,
118
+ "grad_norm": 0.0473485067486763,
119
+ "kl": 0.026468778911389802,
120
  "learning_rate": 2.998523534736735e-06,
121
+ "loss": 0.0,
122
+ "reward": 0.6842105420012223,
123
+ "reward_std": 0.3472982666994396,
124
+ "rewards/accuracy_reward": 0.6842105420012223,
125
  "rewards/format_reward": 0.0,
126
  "step": 45
127
  },
128
  {
129
+ "completion_length": 628.0421226099918,
130
  "epoch": 0.12666666666666668,
131
+ "grad_norm": 0.11586566269397736,
132
+ "kl": 0.004013121755499589,
133
  "learning_rate": 2.994097045546504e-06,
134
+ "loss": 0.0,
135
+ "reward": 0.7263158050022627,
136
+ "reward_std": 0.3308417533573351,
137
+ "rewards/accuracy_reward": 0.7263158050022627,
138
  "rewards/format_reward": 0.0,
139
  "step": 50
140
  },
141
  {
142
+ "completion_length": 634.6701881810238,
143
  "epoch": 0.13933333333333334,
144
+ "grad_norm": 0.09287750720977783,
145
+ "kl": 0.0033212159809313324,
146
  "learning_rate": 2.986729246506011e-06,
147
+ "loss": 0.0,
148
+ "reward": 0.6912280863837192,
149
+ "reward_std": 0.35596638729697777,
150
+ "rewards/accuracy_reward": 0.6912280863837192,
151
  "rewards/format_reward": 0.0,
152
  "step": 55
153
  },
154
  {
155
+ "completion_length": 607.8982656378495,
156
  "epoch": 0.152,
157
+ "grad_norm": 0.1364029198884964,
158
+ "kl": 0.003622757761101974,
159
  "learning_rate": 2.976434642014389e-06,
160
+ "loss": 0.0,
161
+ "reward": 0.7561403630595458,
162
+ "reward_std": 0.2851124694472865,
163
+ "rewards/accuracy_reward": 0.7543859771992031,
164
+ "rewards/format_reward": 0.0017543860171970569,
165
  "step": 60
166
  },
167
  {
168
+ "completion_length": 605.7438781738281,
169
  "epoch": 0.16466666666666666,
170
+ "grad_norm": 0.0922616571187973,
171
+ "kl": 0.007114209626850329,
172
  "learning_rate": 2.9632334982395456e-06,
173
+ "loss": 0.0,
174
+ "reward": 0.7578947509589948,
175
+ "reward_std": 0.29374928286201074,
176
+ "rewards/accuracy_reward": 0.7578947509589948,
177
  "rewards/format_reward": 0.0,
178
  "step": 65
179
  },
180
  {
181
+ "completion_length": 597.1175623843544,
182
  "epoch": 0.17733333333333334,
183
+ "grad_norm": 0.0,
184
+ "kl": 0.003088178132709704,
185
  "learning_rate": 2.947151803221774e-06,
186
+ "loss": 0.0,
187
+ "reward": 0.7596491361919202,
188
+ "reward_std": 0.2632309951280293,
189
+ "rewards/accuracy_reward": 0.7596491361919202,
190
  "rewards/format_reward": 0.0,
191
  "step": 70
192
  },
193
  {
194
+ "completion_length": 599.9701933208265,
195
  "epoch": 0.19,
196
+ "grad_norm": 0.10758549720048904,
197
+ "kl": 0.003361149838096217,
198
  "learning_rate": 2.928221215713164e-06,
199
+ "loss": 0.0,
200
+ "reward": 0.7666666791627281,
201
+ "reward_std": 0.2686667611724452,
202
+ "rewards/accuracy_reward": 0.7666666791627281,
203
  "rewards/format_reward": 0.0,
204
  "step": 75
205
  },
206
  {
207
+ "completion_length": 597.1456305252879,
208
  "epoch": 0.20266666666666666,
209
+ "grad_norm": 0.06053958460688591,
210
+ "kl": 0.0037446674547697368,
211
  "learning_rate": 2.906479002853542e-06,
212
+ "loss": 0.0,
213
+ "reward": 0.7315789610147476,
214
+ "reward_std": 0.30245108447576824,
215
+ "rewards/accuracy_reward": 0.7315789610147476,
216
  "rewards/format_reward": 0.0,
217
  "step": 80
218
  },
219
  {
220
+ "completion_length": 577.8438778525905,
221
  "epoch": 0.21533333333333332,
222
+ "grad_norm": 0.0840262621641159,
223
+ "kl": 0.004712596692537006,
224
  "learning_rate": 2.8819679668056195e-06,
225
+ "loss": 0.0,
226
+ "reward": 0.7982456276291295,
227
+ "reward_std": 0.26917701614530465,
228
+ "rewards/accuracy_reward": 0.7982456276291295,
229
  "rewards/format_reward": 0.0,
230
  "step": 85
231
  },
232
  {
233
+ "completion_length": 612.2982611405222,
234
  "epoch": 0.228,
235
+ "grad_norm": 0.0,
236
+ "kl": 0.02382647865696957,
237
  "learning_rate": 2.8547363604937856e-06,
238
+ "loss": 0.0,
239
+ "reward": 0.7456140491523241,
240
+ "reward_std": 0.2956205848016237,
241
+ "rewards/accuracy_reward": 0.7456140491523241,
242
  "rewards/format_reward": 0.0,
243
  "step": 90
244
  },
245
  {
246
+ "completion_length": 598.1403680098684,
247
  "epoch": 0.24066666666666667,
248
+ "grad_norm": 0.0,
249
+ "kl": 0.005134462055407072,
250
  "learning_rate": 2.824837792612416e-06,
251
+ "loss": 0.0,
252
+ "reward": 0.7403508889047723,
253
+ "reward_std": 0.2658375608293634,
254
+ "rewards/accuracy_reward": 0.7403508889047723,
255
  "rewards/format_reward": 0.0,
256
  "step": 95
257
  },
258
  {
259
+ "completion_length": 582.6561587685032,
260
  "epoch": 0.25333333333333335,
261
+ "grad_norm": 0.11761737614870071,
262
+ "kl": 0.00578155517578125,
263
  "learning_rate": 2.792331122090709e-06,
264
+ "loss": 0.0,
265
+ "reward": 0.7614035217385543,
266
+ "reward_std": 0.25812496166480214,
267
+ "rewards/accuracy_reward": 0.7614035217385543,
268
  "rewards/format_reward": 0.0,
269
  "step": 100
270
  },
271
  {
272
  "epoch": 0.25333333333333335,
273
+ "eval_completion_length": 576.5513494110107,
274
+ "eval_kl": 0.10147758865356446,
275
+ "eval_loss": 3.062895004291022e-08,
276
+ "eval_reward": 0.6709333465278149,
277
+ "eval_reward_std": 0.3338339452087879,
278
+ "eval_rewards/accuracy_reward": 0.6709333465278149,
279
  "eval_rewards/format_reward": 0.0,
280
+ "eval_runtime": 79047.156,
281
+ "eval_samples_per_second": 0.063,
282
+ "eval_steps_per_second": 0.011,
283
  "step": 100
284
  },
285
  {
286
+ "completion_length": 549.310542056435,
287
  "epoch": 0.266,
288
+ "grad_norm": 0.08108708262443542,
289
+ "kl": 0.07660410027754934,
290
  "learning_rate": 2.7572803422217976e-06,
291
+ "loss": 0.0,
292
+ "reward": 0.8140351000585054,
293
+ "reward_std": 0.2548168144728008,
294
+ "rewards/accuracy_reward": 0.8140351000585054,
295
  "rewards/format_reward": 0.0,
296
  "step": 105
297
  },
298
  {
299
+ "completion_length": 596.6351042094983,
300
  "epoch": 0.2786666666666667,
301
+ "grad_norm": 0.06687445938587189,
302
+ "kl": 0.18131464907997533,
303
  "learning_rate": 2.71975445468425e-06,
304
+ "loss": 0.0,
305
+ "reward": 0.7543859803362897,
306
+ "reward_std": 0.32687413692474365,
307
+ "rewards/accuracy_reward": 0.7543859803362897,
308
  "rewards/format_reward": 0.0,
309
  "step": 110
310
  },
311
  {
312
+ "completion_length": 547.8263289602179,
313
  "epoch": 0.29133333333333333,
314
+ "grad_norm": 0.08061777800321579,
315
+ "kl": 4.457168900339227,
316
  "learning_rate": 2.679827333703964e-06,
317
+ "loss": -0.0,
318
+ "reward": 0.7894736992685418,
319
+ "reward_std": 0.2753143636803878,
320
+ "rewards/accuracy_reward": 0.7894736992685418,
321
  "rewards/format_reward": 0.0,
322
  "step": 115
323
  },
324
  {
325
+ "completion_length": 533.67895989669,
326
  "epoch": 0.304,
327
+ "grad_norm": 0.15190833806991577,
328
+ "kl": 0.005019980982730263,
329
  "learning_rate": 2.637577580623858e-06,
330
+ "loss": 0.0,
331
+ "reward": 0.771929837528028,
332
+ "reward_std": 0.2711241148020092,
333
+ "rewards/accuracy_reward": 0.771929837528028,
334
  "rewards/format_reward": 0.0,
335
  "step": 120
336
  },
337
  {
338
+ "completion_length": 535.5947535464638,
339
  "epoch": 0.31666666666666665,
340
+ "grad_norm": 0.24402964115142822,
341
+ "kl": 0.005212723581414474,
342
  "learning_rate": 2.593088369167671e-06,
343
+ "loss": 0.0,
344
+ "reward": 0.8035087809750908,
345
+ "reward_std": 0.2135340957265151,
346
+ "rewards/accuracy_reward": 0.8035087809750908,
347
  "rewards/format_reward": 0.0,
348
  "step": 125
349
  },
350
  {
351
+ "completion_length": 532.9789643940172,
352
  "epoch": 0.3293333333333333,
353
+ "grad_norm": 0.28009745478630066,
354
+ "kl": 0.005336239463404605,
355
  "learning_rate": 2.5464472817024772e-06,
356
+ "loss": 0.0,
357
+ "reward": 0.8052631699725201,
358
+ "reward_std": 0.24214739736757782,
359
+ "rewards/accuracy_reward": 0.8052631699725201,
360
  "rewards/format_reward": 0.0,
361
  "step": 130
362
  },
363
  {
364
+ "completion_length": 561.8772093120374,
365
  "epoch": 0.342,
366
+ "grad_norm": 0.18070097267627716,
367
+ "kl": 0.004754076505962171,
368
  "learning_rate": 2.497746136822254e-06,
369
+ "loss": 0.0,
370
+ "reward": 0.7771929958933278,
371
+ "reward_std": 0.27938908150321556,
372
+ "rewards/accuracy_reward": 0.7771929958933278,
373
  "rewards/format_reward": 0.0,
374
  "step": 135
375
  },
376
  {
377
+ "completion_length": 578.6684380782278,
378
  "epoch": 0.3546666666666667,
379
+ "grad_norm": 0.10906434804201126,
380
+ "kl": 0.005059493215460526,
381
  "learning_rate": 2.4470808085919304e-06,
382
+ "loss": 0.0,
383
+ "reward": 0.7631579088537317,
384
+ "reward_std": 0.3001742513556229,
385
+ "rewards/accuracy_reward": 0.7631579088537317,
386
  "rewards/format_reward": 0.0,
387
  "step": 140
388
  },
389
  {
390
+ "completion_length": 602.6772094726563,
391
  "epoch": 0.36733333333333335,
392
+ "grad_norm": 0.05584894120693207,
393
+ "kl": 0.004565590306332237,
394
  "learning_rate": 2.3945510378077523e-06,
395
+ "loss": 0.0,
396
+ "reward": 0.7842105371387381,
397
+ "reward_std": 0.2481692125922755,
398
+ "rewards/accuracy_reward": 0.7842105371387381,
399
  "rewards/format_reward": 0.0,
400
  "step": 145
401
  },
402
  {
403
+ "completion_length": 580.9210699784128,
404
  "epoch": 0.38,
405
+ "grad_norm": 0.0609310120344162,
406
+ "kl": 0.005594434236225329,
407
  "learning_rate": 2.340260235645519e-06,
408
+ "loss": 0.0,
409
+ "reward": 0.7368421157723979,
410
+ "reward_std": 0.26106126277070296,
411
+ "rewards/accuracy_reward": 0.7368421157723979,
412
  "rewards/format_reward": 0.0,
413
  "step": 150
414
  },
415
  {
416
+ "completion_length": 525.4000133313631,
417
  "epoch": 0.39266666666666666,
418
+ "grad_norm": 0.06953240185976028,
419
+ "kl": 0.007059599223889802,
420
  "learning_rate": 2.2843152800832416e-06,
421
+ "loss": 0.0,
422
+ "reward": 0.8035087816025082,
423
+ "reward_std": 0.2169914556177039,
424
+ "rewards/accuracy_reward": 0.8035087816025082,
425
  "rewards/format_reward": 0.0,
426
  "step": 155
427
  },
428
  {
429
+ "completion_length": 559.4386140522204,
430
  "epoch": 0.4053333333333333,
431
+ "grad_norm": 0.08336005359888077,
432
+ "kl": 0.007145851536800986,
433
  "learning_rate": 2.2268263054989753e-06,
434
+ "loss": 0.0,
435
+ "reward": 0.7824561511215411,
436
+ "reward_std": 0.23924477821902224,
437
+ "rewards/accuracy_reward": 0.7824561511215411,
438
  "rewards/format_reward": 0.0,
439
  "step": 160
440
  },
441
  {
442
+ "completion_length": 546.5772112394634,
443
  "epoch": 0.418,
444
+ "grad_norm": 0.0791773647069931,
445
+ "kl": 0.0061695299650493425,
446
  "learning_rate": 2.167906485858047e-06,
447
+ "loss": 0.0,
448
+ "reward": 0.803508786308138,
449
+ "reward_std": 0.25981574560466564,
450
+ "rewards/accuracy_reward": 0.803508786308138,
451
  "rewards/format_reward": 0.0,
452
  "step": 165
453
  },
454
  {
455
+ "completion_length": 568.9280845240543,
456
  "epoch": 0.43066666666666664,
457
+ "grad_norm": 0.11959535628557205,
458
+ "kl": 0.007060161389802632,
459
  "learning_rate": 2.1076718119164804e-06,
460
+ "loss": 0.0,
461
+ "reward": 0.7736842216629731,
462
+ "reward_std": 0.2649867666395087,
463
+ "rewards/accuracy_reward": 0.7736842216629731,
464
  "rewards/format_reward": 0.0,
465
  "step": 170
466
  },
467
  {
468
+ "completion_length": 550.3982609799034,
469
  "epoch": 0.44333333333333336,
470
+ "grad_norm": 0.0,
471
+ "kl": 0.007817157946134868,
472
  "learning_rate": 2.0462408628792335e-06,
473
+ "loss": 0.0,
474
+ "reward": 0.7824561511215411,
475
+ "reward_std": 0.253275244800668,
476
+ "rewards/accuracy_reward": 0.7824561511215411,
477
  "rewards/format_reward": 0.0,
478
  "step": 175
479
  },
480
  {
481
+ "completion_length": 586.7596660413241,
482
  "epoch": 0.456,
483
+ "grad_norm": 0.06831427663564682,
484
+ "kl": 0.0069172106291118425,
485
  "learning_rate": 1.9837345729627633e-06,
486
+ "loss": 0.0,
487
+ "reward": 0.7438596612528751,
488
+ "reward_std": 0.2811869652647721,
489
+ "rewards/accuracy_reward": 0.7438596612528751,
490
  "rewards/format_reward": 0.0,
491
  "step": 180
492
  },
493
  {
494
+ "completion_length": 583.1228234542043,
495
  "epoch": 0.4686666666666667,
496
+ "grad_norm": 0.10550453513860703,
497
+ "kl": 0.006921226099917763,
498
  "learning_rate": 1.9202759933214665e-06,
499
+ "loss": 0.0,
500
+ "reward": 0.7631579093242946,
501
+ "reward_std": 0.29657848289138394,
502
+ "rewards/accuracy_reward": 0.7631579093242946,
503
  "rewards/format_reward": 0.0,
504
  "step": 185
505
  },
506
  {
507
+ "completion_length": 528.0912424187911,
508
  "epoch": 0.48133333333333334,
509
+ "grad_norm": 0.0677071213722229,
510
+ "kl": 0.009208438270970395,
511
  "learning_rate": 1.8559900498066726e-06,
512
+ "loss": 0.0,
513
+ "reward": 0.8140350991173795,
514
+ "reward_std": 0.22778719381282203,
515
+ "rewards/accuracy_reward": 0.8140350991173795,
516
  "rewards/format_reward": 0.0,
517
  "step": 190
518
  },
519
  {
520
+ "completion_length": 550.7807165848582,
521
  "epoch": 0.494,
522
+ "grad_norm": 0.18718236684799194,
523
+ "kl": 0.008406307822779605,
524
  "learning_rate": 1.7910032970350677e-06,
525
+ "loss": 0.0,
526
+ "reward": 0.7789473815968162,
527
+ "reward_std": 0.2719327967417868,
528
+ "rewards/accuracy_reward": 0.7789473815968162,
529
  "rewards/format_reward": 0.0,
530
  "step": 195
531
  },
532
  {
533
+ "completion_length": 551.1315929212068,
534
  "epoch": 0.5066666666666667,
535
+ "grad_norm": 0.08260176330804825,
536
+ "kl": 0.00812426115337171,
537
  "learning_rate": 1.7254436692507058e-06,
538
+ "loss": 0.0,
539
+ "reward": 0.8070175547348826,
540
+ "reward_std": 0.23249007557567797,
541
+ "rewards/accuracy_reward": 0.8070175547348826,
542
  "rewards/format_reward": 0.0,
543
  "step": 200
544
  },
545
  {
546
  "epoch": 0.5066666666666667,
547
+ "eval_completion_length": 553.8582824371338,
548
+ "eval_kl": 0.02361789245605469,
549
+ "eval_loss": -4.30000654887408e-05,
550
+ "eval_reward": 0.6822333461046219,
551
+ "eval_reward_std": 0.32350232841968535,
552
+ "eval_rewards/accuracy_reward": 0.6822333461046219,
553
  "eval_rewards/format_reward": 0.0,
554
+ "eval_runtime": 76484.7991,
555
+ "eval_samples_per_second": 0.065,
556
+ "eval_steps_per_second": 0.011,
557
  "step": 200
558
  },
559
  {
560
+ "completion_length": 570.7842259457237,
561
  "epoch": 0.5193333333333333,
562
+ "grad_norm": 0.14592121541500092,
563
+ "kl": 0.024847893965871712,
564
  "learning_rate": 1.6594402284710481e-06,
565
+ "loss": 0.0,
566
+ "reward": 0.7561403608635853,
567
+ "reward_std": 0.24963736345893459,
568
+ "rewards/accuracy_reward": 0.7561403608635853,
569
  "rewards/format_reward": 0.0,
570
  "step": 205
571
  },
572
  {
573
+ "completion_length": 556.5035239771793,
574
  "epoch": 0.532,
575
+ "grad_norm": 0.06877300888299942,
576
+ "kl": 0.011327321905838815,
577
  "learning_rate": 1.593122910412851e-06,
578
+ "loss": 0.0,
579
+ "reward": 0.8210526436567307,
580
+ "reward_std": 0.22859587669372558,
581
+ "rewards/accuracy_reward": 0.8210526436567307,
582
  "rewards/format_reward": 0.0,
583
  "step": 210
584
  },
585
  {
586
+ "completion_length": 579.1263334575452,
587
  "epoch": 0.5446666666666666,
588
+ "grad_norm": 0.0,
589
+ "kl": 0.007683683696546053,
590
  "learning_rate": 1.5266222686980693e-06,
591
+ "loss": 0.0,
592
+ "reward": 0.7561403643143805,
593
+ "reward_std": 0.2832074833543677,
594
+ "rewards/accuracy_reward": 0.7561403643143805,
595
  "rewards/format_reward": 0.0,
596
  "step": 215
597
  },
598
  {
599
+ "completion_length": 588.4649293598376,
600
  "epoch": 0.5573333333333333,
601
+ "grad_norm": 0.07735468447208405,
602
+ "kl": 0.007682238127055921,
603
  "learning_rate": 1.460069217843338e-06,
604
+ "loss": 0.0,
605
+ "reward": 0.7333333465613817,
606
+ "reward_std": 0.31152473405787817,
607
+ "rewards/accuracy_reward": 0.7333333465613817,
608
  "rewards/format_reward": 0.0,
609
  "step": 220
610
  },
611
  {
612
+ "completion_length": 582.8508938438014,
613
  "epoch": 0.57,
614
+ "grad_norm": 0.04747384414076805,
615
+ "kl": 0.008666349712171053,
616
  "learning_rate": 1.3935947755389924e-06,
617
+ "loss": 0.0,
618
+ "reward": 0.7456140472700722,
619
+ "reward_std": 0.27623014795152767,
620
+ "rewards/accuracy_reward": 0.7456140472700722,
621
  "rewards/format_reward": 0.0,
622
  "step": 225
623
  },
624
  {
625
+ "completion_length": 541.6929962158204,
626
  "epoch": 0.5826666666666667,
627
+ "grad_norm": 0.0,
628
+ "kl": 0.009119616056743421,
629
  "learning_rate": 1.3273298047249756e-06,
630
+ "loss": 0.0,
631
+ "reward": 0.7964912389454089,
632
+ "reward_std": 0.2623067799367403,
633
+ "rewards/accuracy_reward": 0.7964912389454089,
634
  "rewards/format_reward": 0.0,
635
  "step": 230
636
  },
637
  {
638
+ "completion_length": 555.8666826750102,
639
  "epoch": 0.5953333333333334,
640
+ "grad_norm": 0.1013207659125328,
641
+ "kl": 0.008411447625411184,
642
  "learning_rate": 1.2614047559713923e-06,
643
+ "loss": 0.0,
644
+ "reward": 0.8263158042179911,
645
+ "reward_std": 0.25948601145493355,
646
+ "rewards/accuracy_reward": 0.8263158042179911,
647
  "rewards/format_reward": 0.0,
648
  "step": 235
649
  },
650
  {
651
+ "completion_length": 564.3929960552015,
652
  "epoch": 0.608,
653
+ "grad_norm": 0.08883073180913925,
654
+ "kl": 0.008549579821134868,
655
  "learning_rate": 1.1959494106708598e-06,
656
+ "loss": 0.0,
657
+ "reward": 0.792982468950121,
658
+ "reward_std": 0.25643417960719056,
659
+ "rewards/accuracy_reward": 0.792982468950121,
660
  "rewards/format_reward": 0.0,
661
  "step": 240
662
  },
663
  {
664
+ "completion_length": 536.9596663625617,
665
  "epoch": 0.6206666666666667,
666
+ "grad_norm": 0.04451132193207741,
667
+ "kl": 0.009548789576480263,
668
  "learning_rate": 1.1310926255482204e-06,
669
+ "loss": 0.0,
670
+ "reward": 0.8052631693451028,
671
+ "reward_std": 0.24199818247242977,
672
+ "rewards/accuracy_reward": 0.8052631693451028,
673
  "rewards/format_reward": 0.0,
674
  "step": 245
675
  },
676
  {
677
+ "completion_length": 544.7193134508635,
678
  "epoch": 0.6333333333333333,
679
+ "grad_norm": 0.0,
680
+ "kl": 0.009332596628289473,
681
  "learning_rate": 1.0669620789905688e-06,
682
+ "loss": 0.0,
683
+ "reward": 0.800000010038677,
684
+ "reward_std": 0.23219164547167326,
685
+ "rewards/accuracy_reward": 0.800000010038677,
686
  "rewards/format_reward": 0.0,
687
  "step": 250
688
  },
689
  {
690
+ "completion_length": 552.8649261474609,
691
  "epoch": 0.646,
692
+ "grad_norm": 0.06963980942964554,
693
+ "kl": 0.009455148797286185,
694
  "learning_rate": 1.0036840196969795e-06,
695
+ "loss": 0.0,
696
+ "reward": 0.807017553009485,
697
+ "reward_std": 0.21750171090427198,
698
+ "rewards/accuracy_reward": 0.807017553009485,
699
  "rewards/format_reward": 0.0,
700
  "step": 255
701
  },
702
  {
703
+ "completion_length": 560.9298404091282,
704
  "epoch": 0.6586666666666666,
705
+ "grad_norm": 0.13915280997753143,
706
+ "kl": 0.008615754780016447,
707
  "learning_rate": 9.413830181427508e-07,
708
+ "loss": 0.0,
709
+ "reward": 0.7894736975431442,
710
+ "reward_std": 0.26591097994854573,
711
+ "rewards/accuracy_reward": 0.7894736975431442,
712
  "rewards/format_reward": 0.0,
713
  "step": 260
714
  },
715
  {
716
+ "completion_length": 583.3000167043585,
717
  "epoch": 0.6713333333333333,
718
+ "grad_norm": 0.0,
719
+ "kl": 0.008441322728207237,
720
  "learning_rate": 8.801817213474331e-07,
721
+ "loss": 0.0,
722
+ "reward": 0.7508772039099744,
723
+ "reward_std": 0.2809643321915677,
724
+ "rewards/accuracy_reward": 0.7508772039099744,
725
  "rewards/format_reward": 0.0,
726
  "step": 265
727
  },
728
  {
729
+ "completion_length": 531.2859802246094,
730
  "epoch": 0.684,
731
+ "grad_norm": 0.05701286345720291,
732
+ "kl": 0.00885218570106908,
733
  "learning_rate": 8.202006114294044e-07,
734
+ "loss": 0.0,
735
+ "reward": 0.7877193099574039,
736
+ "reward_std": 0.2481692119648582,
737
+ "rewards/accuracy_reward": 0.7877193099574039,
738
  "rewards/format_reward": 0.0,
739
  "step": 270
740
  },
741
  {
742
+ "completion_length": 531.2403676886307,
743
  "epoch": 0.6966666666666667,
744
+ "grad_norm": 0.05670388042926788,
745
+ "kl": 0.009579307154605263,
746
  "learning_rate": 7.615577684223272e-07,
747
+ "loss": 0.0,
748
+ "reward": 0.8175438716223365,
749
+ "reward_std": 0.23557321397881759,
750
+ "rewards/accuracy_reward": 0.8175438716223365,
751
  "rewards/format_reward": 0.0,
752
  "step": 275
753
  },
754
  {
755
+ "completion_length": 544.3842264275802,
756
  "epoch": 0.7093333333333334,
757
+ "grad_norm": 0.0,
758
+ "kl": 0.009712299547697369,
759
  "learning_rate": 7.043686378203864e-07,
760
+ "loss": 0.0,
761
+ "reward": 0.7701754490011616,
762
+ "reward_std": 0.25272287663660553,
763
+ "rewards/accuracy_reward": 0.7701754490011616,
764
  "rewards/format_reward": 0.0,
765
  "step": 280
766
  },
767
  {
768
+ "completion_length": 571.7438746402138,
769
  "epoch": 0.722,
770
+ "grad_norm": 0.08127343654632568,
771
+ "kl": 0.008696064196134869,
772
  "learning_rate": 6.487458033099425e-07,
773
+ "loss": 0.0,
774
+ "reward": 0.7701754521382482,
775
+ "reward_std": 0.2902606151605907,
776
+ "rewards/accuracy_reward": 0.7701754521382482,
777
  "rewards/format_reward": 0.0,
778
  "step": 285
779
  },
780
  {
781
+ "completion_length": 532.1596609015214,
782
  "epoch": 0.7346666666666667,
783
+ "grad_norm": 0.072137750685215,
784
+ "kl": 0.008551828484786184,
785
  "learning_rate": 5.947987651349942e-07,
786
+ "loss": 0.0,
787
+ "reward": 0.8070175544211738,
788
+ "reward_std": 0.21213936178307785,
789
+ "rewards/accuracy_reward": 0.8070175544211738,
790
  "rewards/format_reward": 0.0,
791
  "step": 290
792
  },
793
  {
794
+ "completion_length": 535.1210667660362,
795
  "epoch": 0.7473333333333333,
796
+ "grad_norm": 0.07434926927089691,
797
+ "kl": 0.008226896587171052,
798
  "learning_rate": 5.426337245327703e-07,
799
+ "loss": 0.0,
800
+ "reward": 0.8157894832523246,
801
+ "reward_std": 0.22407589335190622,
802
+ "rewards/accuracy_reward": 0.8157894832523246,
803
  "rewards/format_reward": 0.0,
804
  "step": 295
805
  },
806
  {
807
+ "completion_length": 536.6631729929071,
808
  "epoch": 0.76,
809
+ "grad_norm": 0.08616889268159866,
810
+ "kl": 0.010335339997944079,
811
  "learning_rate": 4.923533746638108e-07,
812
+ "loss": 0.0,
813
+ "reward": 0.8105263270829853,
814
+ "reward_std": 0.24915841535518043,
815
+ "rewards/accuracy_reward": 0.8105263270829853,
816
  "rewards/format_reward": 0.0,
817
  "step": 300
818
  },
819
  {
820
  "epoch": 0.76,
821
+ "eval_completion_length": 545.0558823242187,
822
+ "eval_kl": 0.6695549743652344,
823
+ "eval_loss": 3.003481197083602e-08,
824
+ "eval_reward": 0.6868333460628986,
825
+ "eval_reward_std": 0.3212484232842922,
826
+ "eval_rewards/accuracy_reward": 0.6868333460628986,
827
  "eval_rewards/format_reward": 0.0,
828
+ "eval_runtime": 75622.6169,
829
+ "eval_samples_per_second": 0.066,
830
+ "eval_steps_per_second": 0.011,
831
  "step": 300
832
  },
833
  {
834
+ "completion_length": 574.1017703407689,
835
  "epoch": 0.7726666666666666,
836
+ "grad_norm": 0.0810866504907608,
837
+ "kl": 0.02553052400287829,
838
  "learning_rate": 4.440566984481256e-07,
839
+ "loss": 0.0,
840
+ "reward": 0.7666666799469998,
841
+ "reward_std": 0.27946487539692927,
842
+ "rewards/accuracy_reward": 0.7666666799469998,
843
  "rewards/format_reward": 0.0,
844
  "step": 305
845
  },
846
  {
847
+ "completion_length": 550.2579098350124,
848
  "epoch": 0.7853333333333333,
849
+ "grad_norm": 0.09656031429767609,
850
+ "kl": 0.009140657123766447,
851
  "learning_rate": 3.978387737053994e-07,
852
+ "loss": 0.0,
853
+ "reward": 0.7824561517489584,
854
+ "reward_std": 0.23484032687387968,
855
+ "rewards/accuracy_reward": 0.7824561517489584,
856
  "rewards/format_reward": 0.0,
857
  "step": 310
858
  },
859
  {
860
+ "completion_length": 569.4912441855982,
861
  "epoch": 0.798,
862
+ "grad_norm": 0.094791479408741,
863
+ "kl": 0.009346088610197369,
864
  "learning_rate": 3.5379058598286167e-07,
865
+ "loss": 0.0,
866
+ "reward": 0.7666666777510392,
867
+ "reward_std": 0.25904074530852467,
868
+ "rewards/accuracy_reward": 0.7666666777510392,
869
  "rewards/format_reward": 0.0,
870
  "step": 315
871
  },
872
  {
873
+ "completion_length": 579.2614203202097,
874
  "epoch": 0.8106666666666666,
875
+ "grad_norm": 0.10730049759149551,
876
+ "kl": 4.134726434004934,
877
  "learning_rate": 3.119988494392894e-07,
878
+ "loss": 0.0,
879
+ "reward": 0.7508772053216634,
880
+ "reward_std": 0.2928671808619248,
881
+ "rewards/accuracy_reward": 0.7508772053216634,
882
  "rewards/format_reward": 0.0,
883
  "step": 320
884
  },
885
  {
886
+ "completion_length": 556.2263304057874,
887
  "epoch": 0.8233333333333334,
888
+ "grad_norm": 0.07251843065023422,
889
+ "kl": 0.009259033203125,
890
  "learning_rate": 2.725458361377465e-07,
891
+ "loss": 0.0,
892
+ "reward": 0.7736842222903905,
893
+ "reward_std": 0.24773237705230713,
894
+ "rewards/accuracy_reward": 0.7736842222903905,
895
  "rewards/format_reward": 0.0,
896
  "step": 325
897
  },
898
  {
899
+ "completion_length": 537.1491384405839,
900
  "epoch": 0.836,
901
+ "grad_norm": 0.0,
902
+ "kl": 0.009849789268092105,
903
  "learning_rate": 2.3550921408312737e-07,
904
+ "loss": 0.0,
905
+ "reward": 0.8140350983331078,
906
+ "reward_std": 0.23072349554614016,
907
+ "rewards/accuracy_reward": 0.8140350983331078,
908
  "rewards/format_reward": 0.0,
909
  "step": 330
910
  },
911
  {
912
+ "completion_length": 547.5140491686369,
913
  "epoch": 0.8486666666666667,
914
+ "grad_norm": 0.08143898099660873,
915
+ "kl": 0.010343531558388158,
916
  "learning_rate": 2.0096189432334195e-07,
917
+ "loss": 0.0,
918
+ "reward": 0.7824561493961435,
919
+ "reward_std": 0.23674531171196386,
920
+ "rewards/accuracy_reward": 0.7824561493961435,
921
  "rewards/format_reward": 0.0,
922
  "step": 335
923
  },
924
  {
925
+ "completion_length": 541.5175582082649,
926
  "epoch": 0.8613333333333333,
927
+ "grad_norm": 0.15802732110023499,
928
+ "kl": 0.009666683799342105,
929
  "learning_rate": 1.6897188741514286e-07,
930
+ "loss": 0.0,
931
+ "reward": 0.7666666757119329,
932
+ "reward_std": 0.24941472881718685,
933
+ "rewards/accuracy_reward": 0.7666666757119329,
934
  "rewards/format_reward": 0.0,
935
  "step": 340
936
  },
937
  {
938
+ "completion_length": 566.2824713456004,
939
  "epoch": 0.874,
940
+ "grad_norm": 0.08206634968519211,
941
+ "kl": 0.009324886924342106,
942
  "learning_rate": 1.396021695371582e-07,
943
+ "loss": 0.0,
944
+ "reward": 0.7543859756306598,
945
+ "reward_std": 0.23858530960584942,
946
+ "rewards/accuracy_reward": 0.7543859756306598,
947
  "rewards/format_reward": 0.0,
948
  "step": 345
949
  },
950
  {
951
+ "completion_length": 567.8473815917969,
952
  "epoch": 0.8866666666666667,
953
+ "grad_norm": 0.06388476490974426,
954
+ "kl": 0.010038034539473684,
955
  "learning_rate": 1.1291055851370623e-07,
956
+ "loss": 0.0,
957
+ "reward": 0.7491228202455922,
958
+ "reward_std": 0.2948876986378118,
959
+ "rewards/accuracy_reward": 0.7491228202455922,
960
  "rewards/format_reward": 0.0,
961
  "step": 350
962
  },
963
  {
964
+ "completion_length": 576.67545663934,
965
  "epoch": 0.8993333333333333,
966
+ "grad_norm": 0.058260105550289154,
967
+ "kl": 0.008216456363075659,
968
  "learning_rate": 8.894959999345015e-08,
969
+ "loss": 0.0,
970
+ "reward": 0.8017543960558741,
971
+ "reward_std": 0.23333006344343488,
972
+ "rewards/accuracy_reward": 0.8017543960558741,
973
  "rewards/format_reward": 0.0,
974
  "step": 355
975
  },
976
  {
977
+ "completion_length": 549.091246434262,
978
  "epoch": 0.912,
979
+ "grad_norm": 0.10874364525079727,
980
+ "kl": 0.00869959781044408,
981
  "learning_rate": 6.776646400696212e-08,
982
+ "loss": 0.0,
983
+ "reward": 0.796491244749019,
984
+ "reward_std": 0.3010250455454776,
985
+ "rewards/accuracy_reward": 0.796491244749019,
986
  "rewards/format_reward": 0.0,
987
  "step": 360
988
  },
989
  {
990
+ "completion_length": 560.8719421386719,
991
  "epoch": 0.9246666666666666,
992
+ "grad_norm": 0.10691142827272415,
993
+ "kl": 0.009839509662828948,
994
  "learning_rate": 4.940285210684375e-08,
995
+ "loss": 0.0,
996
+ "reward": 0.7964912431804757,
997
+ "reward_std": 0.28669615042837043,
998
+ "rewards/accuracy_reward": 0.7964912431804757,
999
  "rewards/format_reward": 0.0,
1000
  "step": 365
1001
  },
1002
  {
1003
+ "completion_length": 570.056158126028,
1004
  "epoch": 0.9373333333333334,
1005
+ "grad_norm": 0.14333242177963257,
1006
+ "kl": 0.008819580078125,
1007
  "learning_rate": 3.389491527319999e-08,
1008
+ "loss": 0.0,
1009
+ "reward": 0.7631579105791293,
1010
+ "reward_std": 0.3113442129210422,
1011
+ "rewards/accuracy_reward": 0.7631579105791293,
1012
  "rewards/format_reward": 0.0,
1013
  "step": 370
1014
  },
1015
  {
1016
+ "completion_length": 581.7122971384149,
1017
  "epoch": 0.95,
1018
+ "grad_norm": 0.12897399067878723,
1019
+ "kl": 0.009299830386513159,
1020
  "learning_rate": 2.127318274608381e-08,
1021
+ "loss": 0.0,
1022
+ "reward": 0.759649133211688,
1023
+ "reward_std": 0.24736052971137198,
1024
+ "rewards/accuracy_reward": 0.759649133211688,
1025
  "rewards/format_reward": 0.0,
1026
  "step": 375
1027
  },
1028
  {
1029
+ "completion_length": 519.6631728322883,
1030
  "epoch": 0.9626666666666667,
1031
+ "grad_norm": 0.046151068061590195,
1032
+ "kl": 0.018807340923108554,
1033
  "learning_rate": 1.1562501925013125e-08,
1034
+ "loss": 0.0,
1035
+ "reward": 0.8105263286515286,
1036
+ "reward_std": 0.24725342863484434,
1037
+ "rewards/accuracy_reward": 0.8105263286515286,
1038
  "rewards/format_reward": 0.0,
1039
  "step": 380
1040
  },
1041
  {
1042
+ "completion_length": 522.570188984118,
1043
  "epoch": 0.9753333333333334,
1044
+ "grad_norm": 0.10620765388011932,
1045
+ "kl": 0.00986279939350329,
1046
  "learning_rate": 4.781989453874814e-09,
1047
+ "loss": 0.0,
1048
+ "reward": 0.7964912391022632,
1049
+ "reward_std": 0.2455626440675635,
1050
+ "rewards/accuracy_reward": 0.7964912391022632,
1051
  "rewards/format_reward": 0.0,
1052
  "step": 385
1053
  },
1054
  {
1055
+ "completion_length": 536.8315922787315,
1056
  "epoch": 0.988,
1057
+ "grad_norm": 0.09291122853755951,
1058
+ "kl": 0.010211502878289473,
1059
  "learning_rate": 9.44993587509657e-10,
1060
+ "loss": 0.0,
1061
+ "reward": 0.8070175572445518,
1062
+ "reward_std": 0.2619433641433716,
1063
+ "rewards/accuracy_reward": 0.8070175572445518,
1064
  "rewards/format_reward": 0.0,
1065
  "step": 390
1066
  },
1067
  {
1068
+ "completion_length": 567.2982639513517,
1069
  "epoch": 0.9981333333333333,
1070
+ "kl": 0.00917715775339227,
1071
+ "reward": 0.7828947472337046,
1072
+ "reward_std": 0.2575972625299504,
1073
+ "rewards/accuracy_reward": 0.7828947472337046,
1074
  "rewards/format_reward": 0.0,
1075
  "step": 394,
1076
  "total_flos": 0.0,
1077
+ "train_loss": 0.0008256121124141636,
1078
+ "train_runtime": 419098.9196,
1079
+ "train_samples_per_second": 0.018,
1080
  "train_steps_per_second": 0.001
1081
  }
1082
  ],
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:6f942f0096b55244d30974e5a2a781aa646e662406e9dee451d546238cbf9b6c
3
  size 7480
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:84f3f7010102b0e305468c1cf34da3672c32048c60d1693fa81fc6817dc89fe5
3
  size 7480