Model save
Browse files- README.md +2 -4
- all_results.json +3 -8
- config.json +1 -1
- model-00001-of-00004.safetensors +1 -1
- model-00002-of-00004.safetensors +1 -1
- model-00003-of-00004.safetensors +1 -1
- model-00004-of-00004.safetensors +1 -1
- train_results.json +3 -3
- trainer_state.json +579 -579
- training_args.bin +1 -1
README.md
CHANGED
@@ -1,11 +1,9 @@
|
|
1 |
---
|
2 |
base_model: Qwen/Qwen2.5-Math-7B
|
3 |
-
datasets: DigitalLearningGmbH/MATH-lighteval
|
4 |
library_name: transformers
|
5 |
model_name: Qwen-2.5-7B-Simple-RL
|
6 |
tags:
|
7 |
- generated_from_trainer
|
8 |
-
- open-r1
|
9 |
- trl
|
10 |
- grpo
|
11 |
licence: license
|
@@ -13,7 +11,7 @@ licence: license
|
|
13 |
|
14 |
# Model Card for Qwen-2.5-7B-Simple-RL
|
15 |
|
16 |
-
This model is a fine-tuned version of [Qwen/Qwen2.5-Math-7B](https://huggingface.co/Qwen/Qwen2.5-Math-7B)
|
17 |
It has been trained using [TRL](https://github.com/huggingface/trl).
|
18 |
|
19 |
## Quick start
|
@@ -29,7 +27,7 @@ print(output["generated_text"])
|
|
29 |
|
30 |
## Training procedure
|
31 |
|
32 |
-
[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/jun-liang-chen-the-hong-kong-polytechnic-university/huggingface/runs/
|
33 |
|
34 |
|
35 |
This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).
|
|
|
1 |
---
|
2 |
base_model: Qwen/Qwen2.5-Math-7B
|
|
|
3 |
library_name: transformers
|
4 |
model_name: Qwen-2.5-7B-Simple-RL
|
5 |
tags:
|
6 |
- generated_from_trainer
|
|
|
7 |
- trl
|
8 |
- grpo
|
9 |
licence: license
|
|
|
11 |
|
12 |
# Model Card for Qwen-2.5-7B-Simple-RL
|
13 |
|
14 |
+
This model is a fine-tuned version of [Qwen/Qwen2.5-Math-7B](https://huggingface.co/Qwen/Qwen2.5-Math-7B).
|
15 |
It has been trained using [TRL](https://github.com/huggingface/trl).
|
16 |
|
17 |
## Quick start
|
|
|
27 |
|
28 |
## Training procedure
|
29 |
|
30 |
+
[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/jun-liang-chen-the-hong-kong-polytechnic-university/huggingface/runs/ye8fls8s)
|
31 |
|
32 |
|
33 |
This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).
|
all_results.json
CHANGED
@@ -1,13 +1,8 @@
|
|
1 |
{
|
2 |
-
"eval_loss": 0.07094848901033401,
|
3 |
-
"eval_runtime": 47796.4045,
|
4 |
-
"eval_samples": 5000,
|
5 |
-
"eval_samples_per_second": 0.105,
|
6 |
-
"eval_steps_per_second": 0.017,
|
7 |
"total_flos": 0.0,
|
8 |
-
"train_loss": 0.
|
9 |
-
"train_runtime":
|
10 |
"train_samples": 7500,
|
11 |
-
"train_samples_per_second": 0.
|
12 |
"train_steps_per_second": 0.001
|
13 |
}
|
|
|
1 |
{
|
|
|
|
|
|
|
|
|
|
|
2 |
"total_flos": 0.0,
|
3 |
+
"train_loss": 0.0008256121124141636,
|
4 |
+
"train_runtime": 419098.9196,
|
5 |
"train_samples": 7500,
|
6 |
+
"train_samples_per_second": 0.018,
|
7 |
"train_steps_per_second": 0.001
|
8 |
}
|
config.json
CHANGED
@@ -23,7 +23,7 @@
|
|
23 |
"tie_word_embeddings": false,
|
24 |
"torch_dtype": "bfloat16",
|
25 |
"transformers_version": "4.49.0.dev0",
|
26 |
-
"use_cache":
|
27 |
"use_mrope": false,
|
28 |
"use_sliding_window": false,
|
29 |
"vocab_size": 152064
|
|
|
23 |
"tie_word_embeddings": false,
|
24 |
"torch_dtype": "bfloat16",
|
25 |
"transformers_version": "4.49.0.dev0",
|
26 |
+
"use_cache": false,
|
27 |
"use_mrope": false,
|
28 |
"use_sliding_window": false,
|
29 |
"vocab_size": 152064
|
model-00001-of-00004.safetensors
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
size 4877660776
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:3d2460834ab32758c7929fd2127f4764dda55d37b2ed228b7d65398fdd13aa3f
|
3 |
size 4877660776
|
model-00002-of-00004.safetensors
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
size 4932751008
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:bcb46f0ac1dcead4a1ccca541f796c45a1d7d22a42f37e19cdcdf1b60bc0583c
|
3 |
size 4932751008
|
model-00003-of-00004.safetensors
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
size 4330865200
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:9ec9437e98f69f030c1e09f702858d79bfacf2ed020e66df9be24b59c445c214
|
3 |
size 4330865200
|
model-00004-of-00004.safetensors
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
size 1089994880
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:fc740e9e6a6d8c353842a1ed8a64535fa9ce1baa0366313305dce8687dfe9175
|
3 |
size 1089994880
|
train_results.json
CHANGED
@@ -1,8 +1,8 @@
|
|
1 |
{
|
2 |
"total_flos": 0.0,
|
3 |
-
"train_loss": 0.
|
4 |
-
"train_runtime":
|
5 |
"train_samples": 7500,
|
6 |
-
"train_samples_per_second": 0.
|
7 |
"train_steps_per_second": 0.001
|
8 |
}
|
|
|
1 |
{
|
2 |
"total_flos": 0.0,
|
3 |
+
"train_loss": 0.0008256121124141636,
|
4 |
+
"train_runtime": 419098.9196,
|
5 |
"train_samples": 7500,
|
6 |
+
"train_samples_per_second": 0.018,
|
7 |
"train_steps_per_second": 0.001
|
8 |
}
|
trainer_state.json
CHANGED
@@ -9,1074 +9,1074 @@
|
|
9 |
"is_world_process_zero": true,
|
10 |
"log_history": [
|
11 |
{
|
12 |
-
"completion_length":
|
13 |
"epoch": 0.012666666666666666,
|
14 |
-
"grad_norm": 0.
|
15 |
-
"kl": 0.
|
16 |
"learning_rate": 3.75e-07,
|
17 |
"loss": 0.0,
|
18 |
-
"reward": 0.
|
19 |
-
"reward_std": 0.
|
20 |
-
"rewards/accuracy_reward": 0.
|
21 |
"rewards/format_reward": 0.0,
|
22 |
"step": 5
|
23 |
},
|
24 |
{
|
25 |
-
"completion_length":
|
26 |
"epoch": 0.025333333333333333,
|
27 |
-
"grad_norm": 0.
|
28 |
-
"kl": 0.
|
29 |
"learning_rate": 7.5e-07,
|
30 |
"loss": 0.0,
|
31 |
-
"reward": 0.
|
32 |
-
"reward_std": 0.
|
33 |
-
"rewards/accuracy_reward": 0.
|
34 |
-
"rewards/format_reward": 0.
|
35 |
"step": 10
|
36 |
},
|
37 |
{
|
38 |
-
"completion_length":
|
39 |
"epoch": 0.038,
|
40 |
-
"grad_norm":
|
41 |
-
"kl": 0.
|
42 |
"learning_rate": 1.125e-06,
|
43 |
"loss": 0.0,
|
44 |
-
"reward": 0.
|
45 |
-
"reward_std": 0.
|
46 |
-
"rewards/accuracy_reward": 0.
|
47 |
"rewards/format_reward": 0.0,
|
48 |
"step": 15
|
49 |
},
|
50 |
{
|
51 |
-
"completion_length":
|
52 |
"epoch": 0.050666666666666665,
|
53 |
-
"grad_norm": 0.
|
54 |
-
"kl": 0.
|
55 |
"learning_rate": 1.5e-06,
|
56 |
"loss": 0.0,
|
57 |
-
"reward": 0.
|
58 |
-
"reward_std": 0.
|
59 |
-
"rewards/accuracy_reward": 0.
|
60 |
"rewards/format_reward": 0.0,
|
61 |
"step": 20
|
62 |
},
|
63 |
{
|
64 |
-
"completion_length":
|
65 |
"epoch": 0.06333333333333334,
|
66 |
-
"grad_norm": 0.
|
67 |
-
"kl": 0.
|
68 |
"learning_rate": 1.875e-06,
|
69 |
"loss": 0.0,
|
70 |
-
"reward": 0.
|
71 |
-
"reward_std": 0.
|
72 |
-
"rewards/accuracy_reward": 0.
|
73 |
"rewards/format_reward": 0.0,
|
74 |
"step": 25
|
75 |
},
|
76 |
{
|
77 |
-
"completion_length":
|
78 |
"epoch": 0.076,
|
79 |
-
"grad_norm": 0.
|
80 |
-
"kl": 0.
|
81 |
"learning_rate": 2.25e-06,
|
82 |
-
"loss": 0.
|
83 |
-
"reward": 0.
|
84 |
-
"reward_std": 0.
|
85 |
-
"rewards/accuracy_reward": 0.
|
86 |
"rewards/format_reward": 0.0,
|
87 |
"step": 30
|
88 |
},
|
89 |
{
|
90 |
-
"completion_length":
|
91 |
"epoch": 0.08866666666666667,
|
92 |
-
"grad_norm": 0.
|
93 |
-
"kl": 0.
|
94 |
"learning_rate": 2.6250000000000003e-06,
|
95 |
-
"loss": 0.
|
96 |
-
"reward": 0.
|
97 |
-
"reward_std": 0.
|
98 |
-
"rewards/accuracy_reward": 0.
|
99 |
-
"rewards/format_reward": 0.
|
100 |
"step": 35
|
101 |
},
|
102 |
{
|
103 |
-
"completion_length":
|
104 |
"epoch": 0.10133333333333333,
|
105 |
-
"grad_norm": 0.
|
106 |
-
"kl": 0.
|
107 |
"learning_rate": 3e-06,
|
108 |
-
"loss": 0.
|
109 |
-
"reward": 0.
|
110 |
-
"reward_std": 0.
|
111 |
-
"rewards/accuracy_reward": 0.
|
112 |
"rewards/format_reward": 0.0,
|
113 |
"step": 40
|
114 |
},
|
115 |
{
|
116 |
-
"completion_length":
|
117 |
"epoch": 0.114,
|
118 |
-
"grad_norm": 0.
|
119 |
-
"kl": 0.
|
120 |
"learning_rate": 2.998523534736735e-06,
|
121 |
-
"loss": 0.
|
122 |
-
"reward": 0.
|
123 |
-
"reward_std": 0.
|
124 |
-
"rewards/accuracy_reward": 0.
|
125 |
"rewards/format_reward": 0.0,
|
126 |
"step": 45
|
127 |
},
|
128 |
{
|
129 |
-
"completion_length":
|
130 |
"epoch": 0.12666666666666668,
|
131 |
-
"grad_norm": 0.
|
132 |
-
"kl": 0.
|
133 |
"learning_rate": 2.994097045546504e-06,
|
134 |
-
"loss": 0.
|
135 |
-
"reward": 0.
|
136 |
-
"reward_std": 0.
|
137 |
-
"rewards/accuracy_reward": 0.
|
138 |
"rewards/format_reward": 0.0,
|
139 |
"step": 50
|
140 |
},
|
141 |
{
|
142 |
-
"completion_length":
|
143 |
"epoch": 0.13933333333333334,
|
144 |
-
"grad_norm": 0.
|
145 |
-
"kl": 0.
|
146 |
"learning_rate": 2.986729246506011e-06,
|
147 |
-
"loss": 0.
|
148 |
-
"reward": 0.
|
149 |
-
"reward_std": 0.
|
150 |
-
"rewards/accuracy_reward": 0.
|
151 |
"rewards/format_reward": 0.0,
|
152 |
"step": 55
|
153 |
},
|
154 |
{
|
155 |
-
"completion_length":
|
156 |
"epoch": 0.152,
|
157 |
-
"grad_norm": 0.
|
158 |
-
"kl": 0.
|
159 |
"learning_rate": 2.976434642014389e-06,
|
160 |
-
"loss": 0.
|
161 |
-
"reward": 0.
|
162 |
-
"reward_std": 0.
|
163 |
-
"rewards/accuracy_reward": 0.
|
164 |
-
"rewards/format_reward": 0.
|
165 |
"step": 60
|
166 |
},
|
167 |
{
|
168 |
-
"completion_length":
|
169 |
"epoch": 0.16466666666666666,
|
170 |
-
"grad_norm": 0.
|
171 |
-
"kl": 0.
|
172 |
"learning_rate": 2.9632334982395456e-06,
|
173 |
-
"loss": 0.
|
174 |
-
"reward": 0.
|
175 |
-
"reward_std": 0.
|
176 |
-
"rewards/accuracy_reward": 0.
|
177 |
"rewards/format_reward": 0.0,
|
178 |
"step": 65
|
179 |
},
|
180 |
{
|
181 |
-
"completion_length":
|
182 |
"epoch": 0.17733333333333334,
|
183 |
-
"grad_norm": 0.
|
184 |
-
"kl": 0.
|
185 |
"learning_rate": 2.947151803221774e-06,
|
186 |
-
"loss": 0.
|
187 |
-
"reward": 0.
|
188 |
-
"reward_std": 0.
|
189 |
-
"rewards/accuracy_reward": 0.
|
190 |
"rewards/format_reward": 0.0,
|
191 |
"step": 70
|
192 |
},
|
193 |
{
|
194 |
-
"completion_length":
|
195 |
"epoch": 0.19,
|
196 |
-
"grad_norm": 0.
|
197 |
-
"kl": 0.
|
198 |
"learning_rate": 2.928221215713164e-06,
|
199 |
-
"loss": 0.
|
200 |
-
"reward": 0.
|
201 |
-
"reward_std": 0.
|
202 |
-
"rewards/accuracy_reward": 0.
|
203 |
"rewards/format_reward": 0.0,
|
204 |
"step": 75
|
205 |
},
|
206 |
{
|
207 |
-
"completion_length":
|
208 |
"epoch": 0.20266666666666666,
|
209 |
-
"grad_norm": 0.
|
210 |
-
"kl": 0.
|
211 |
"learning_rate": 2.906479002853542e-06,
|
212 |
-
"loss": 0.
|
213 |
-
"reward": 0.
|
214 |
-
"reward_std": 0.
|
215 |
-
"rewards/accuracy_reward": 0.
|
216 |
"rewards/format_reward": 0.0,
|
217 |
"step": 80
|
218 |
},
|
219 |
{
|
220 |
-
"completion_length":
|
221 |
"epoch": 0.21533333333333332,
|
222 |
-
"grad_norm": 0.
|
223 |
-
"kl": 0.
|
224 |
"learning_rate": 2.8819679668056195e-06,
|
225 |
-
"loss": 0.
|
226 |
-
"reward": 0.
|
227 |
-
"reward_std": 0.
|
228 |
-
"rewards/accuracy_reward": 0.
|
229 |
"rewards/format_reward": 0.0,
|
230 |
"step": 85
|
231 |
},
|
232 |
{
|
233 |
-
"completion_length":
|
234 |
"epoch": 0.228,
|
235 |
-
"grad_norm": 0.
|
236 |
-
"kl":
|
237 |
"learning_rate": 2.8547363604937856e-06,
|
238 |
-
"loss": 0.
|
239 |
-
"reward": 0.
|
240 |
-
"reward_std": 0.
|
241 |
-
"rewards/accuracy_reward": 0.
|
242 |
"rewards/format_reward": 0.0,
|
243 |
"step": 90
|
244 |
},
|
245 |
{
|
246 |
-
"completion_length":
|
247 |
"epoch": 0.24066666666666667,
|
248 |
-
"grad_norm": 0.
|
249 |
-
"kl": 0.
|
250 |
"learning_rate": 2.824837792612416e-06,
|
251 |
-
"loss": 0.
|
252 |
-
"reward": 0.
|
253 |
-
"reward_std": 0.
|
254 |
-
"rewards/accuracy_reward": 0.
|
255 |
"rewards/format_reward": 0.0,
|
256 |
"step": 95
|
257 |
},
|
258 |
{
|
259 |
-
"completion_length":
|
260 |
"epoch": 0.25333333333333335,
|
261 |
-
"grad_norm": 0.
|
262 |
-
"kl": 0.
|
263 |
"learning_rate": 2.792331122090709e-06,
|
264 |
-
"loss": 0.
|
265 |
-
"reward": 0.
|
266 |
-
"reward_std": 0.
|
267 |
-
"rewards/accuracy_reward": 0.
|
268 |
"rewards/format_reward": 0.0,
|
269 |
"step": 100
|
270 |
},
|
271 |
{
|
272 |
"epoch": 0.25333333333333335,
|
273 |
-
"eval_completion_length":
|
274 |
-
"eval_kl": 0.
|
275 |
-
"eval_loss":
|
276 |
-
"eval_reward": 0.
|
277 |
-
"eval_reward_std": 0.
|
278 |
-
"eval_rewards/accuracy_reward": 0.
|
279 |
"eval_rewards/format_reward": 0.0,
|
280 |
-
"eval_runtime":
|
281 |
-
"eval_samples_per_second": 0.
|
282 |
-
"eval_steps_per_second": 0.
|
283 |
"step": 100
|
284 |
},
|
285 |
{
|
286 |
-
"completion_length":
|
287 |
"epoch": 0.266,
|
288 |
-
"grad_norm": 0.
|
289 |
-
"kl": 0.
|
290 |
"learning_rate": 2.7572803422217976e-06,
|
291 |
-
"loss": 0.
|
292 |
-
"reward": 0.
|
293 |
-
"reward_std": 0.
|
294 |
-
"rewards/accuracy_reward": 0.
|
295 |
"rewards/format_reward": 0.0,
|
296 |
"step": 105
|
297 |
},
|
298 |
{
|
299 |
-
"completion_length":
|
300 |
"epoch": 0.2786666666666667,
|
301 |
-
"grad_norm": 0.
|
302 |
-
"kl": 0.
|
303 |
"learning_rate": 2.71975445468425e-06,
|
304 |
-
"loss": 0.
|
305 |
-
"reward": 0.
|
306 |
-
"reward_std": 0.
|
307 |
-
"rewards/accuracy_reward": 0.
|
308 |
"rewards/format_reward": 0.0,
|
309 |
"step": 110
|
310 |
},
|
311 |
{
|
312 |
-
"completion_length":
|
313 |
"epoch": 0.29133333333333333,
|
314 |
-
"grad_norm": 0.
|
315 |
-
"kl":
|
316 |
"learning_rate": 2.679827333703964e-06,
|
317 |
-
"loss": 0.
|
318 |
-
"reward": 0.
|
319 |
-
"reward_std": 0.
|
320 |
-
"rewards/accuracy_reward": 0.
|
321 |
"rewards/format_reward": 0.0,
|
322 |
"step": 115
|
323 |
},
|
324 |
{
|
325 |
-
"completion_length":
|
326 |
"epoch": 0.304,
|
327 |
-
"grad_norm": 0.
|
328 |
-
"kl": 0.
|
329 |
"learning_rate": 2.637577580623858e-06,
|
330 |
-
"loss": 0.
|
331 |
-
"reward": 0.
|
332 |
-
"reward_std": 0.
|
333 |
-
"rewards/accuracy_reward": 0.
|
334 |
"rewards/format_reward": 0.0,
|
335 |
"step": 120
|
336 |
},
|
337 |
{
|
338 |
-
"completion_length":
|
339 |
"epoch": 0.31666666666666665,
|
340 |
-
"grad_norm": 0.
|
341 |
-
"kl": 0.
|
342 |
"learning_rate": 2.593088369167671e-06,
|
343 |
-
"loss": 0.
|
344 |
-
"reward": 0.
|
345 |
-
"reward_std": 0.
|
346 |
-
"rewards/accuracy_reward": 0.
|
347 |
"rewards/format_reward": 0.0,
|
348 |
"step": 125
|
349 |
},
|
350 |
{
|
351 |
-
"completion_length":
|
352 |
"epoch": 0.3293333333333333,
|
353 |
-
"grad_norm": 0.
|
354 |
-
"kl": 0.
|
355 |
"learning_rate": 2.5464472817024772e-06,
|
356 |
-
"loss": 0.
|
357 |
-
"reward": 0.
|
358 |
-
"reward_std": 0.
|
359 |
-
"rewards/accuracy_reward": 0.
|
360 |
"rewards/format_reward": 0.0,
|
361 |
"step": 130
|
362 |
},
|
363 |
{
|
364 |
-
"completion_length":
|
365 |
"epoch": 0.342,
|
366 |
-
"grad_norm": 0.
|
367 |
-
"kl": 0.
|
368 |
"learning_rate": 2.497746136822254e-06,
|
369 |
-
"loss": 0.
|
370 |
-
"reward": 0.
|
371 |
-
"reward_std": 0.
|
372 |
-
"rewards/accuracy_reward": 0.
|
373 |
"rewards/format_reward": 0.0,
|
374 |
"step": 135
|
375 |
},
|
376 |
{
|
377 |
-
"completion_length":
|
378 |
"epoch": 0.3546666666666667,
|
379 |
-
"grad_norm": 0.
|
380 |
-
"kl": 0.
|
381 |
"learning_rate": 2.4470808085919304e-06,
|
382 |
-
"loss": 0.
|
383 |
-
"reward": 0.
|
384 |
-
"reward_std": 0.
|
385 |
-
"rewards/accuracy_reward": 0.
|
386 |
"rewards/format_reward": 0.0,
|
387 |
"step": 140
|
388 |
},
|
389 |
{
|
390 |
-
"completion_length":
|
391 |
"epoch": 0.36733333333333335,
|
392 |
-
"grad_norm": 0.
|
393 |
-
"kl": 0.
|
394 |
"learning_rate": 2.3945510378077523e-06,
|
395 |
-
"loss": 0.
|
396 |
-
"reward": 0.
|
397 |
-
"reward_std": 0.
|
398 |
-
"rewards/accuracy_reward": 0.
|
399 |
"rewards/format_reward": 0.0,
|
400 |
"step": 145
|
401 |
},
|
402 |
{
|
403 |
-
"completion_length":
|
404 |
"epoch": 0.38,
|
405 |
-
"grad_norm": 0.
|
406 |
-
"kl": 0.
|
407 |
"learning_rate": 2.340260235645519e-06,
|
408 |
-
"loss": 0.
|
409 |
-
"reward": 0.
|
410 |
-
"reward_std": 0.
|
411 |
-
"rewards/accuracy_reward": 0.
|
412 |
"rewards/format_reward": 0.0,
|
413 |
"step": 150
|
414 |
},
|
415 |
{
|
416 |
-
"completion_length":
|
417 |
"epoch": 0.39266666666666666,
|
418 |
-
"grad_norm": 0.
|
419 |
-
"kl": 0.
|
420 |
"learning_rate": 2.2843152800832416e-06,
|
421 |
-
"loss": 0.
|
422 |
-
"reward": 0.
|
423 |
-
"reward_std": 0.
|
424 |
-
"rewards/accuracy_reward": 0.
|
425 |
"rewards/format_reward": 0.0,
|
426 |
"step": 155
|
427 |
},
|
428 |
{
|
429 |
-
"completion_length":
|
430 |
"epoch": 0.4053333333333333,
|
431 |
-
"grad_norm": 0.
|
432 |
-
"kl": 0.
|
433 |
"learning_rate": 2.2268263054989753e-06,
|
434 |
-
"loss": 0.
|
435 |
-
"reward": 0.
|
436 |
-
"reward_std": 0.
|
437 |
-
"rewards/accuracy_reward": 0.
|
438 |
"rewards/format_reward": 0.0,
|
439 |
"step": 160
|
440 |
},
|
441 |
{
|
442 |
-
"completion_length":
|
443 |
"epoch": 0.418,
|
444 |
-
"grad_norm": 0.
|
445 |
-
"kl": 0.
|
446 |
"learning_rate": 2.167906485858047e-06,
|
447 |
-
"loss": 0.
|
448 |
-
"reward": 0.
|
449 |
-
"reward_std": 0.
|
450 |
-
"rewards/accuracy_reward": 0.
|
451 |
"rewards/format_reward": 0.0,
|
452 |
"step": 165
|
453 |
},
|
454 |
{
|
455 |
-
"completion_length":
|
456 |
"epoch": 0.43066666666666664,
|
457 |
-
"grad_norm": 0.
|
458 |
-
"kl": 0.
|
459 |
"learning_rate": 2.1076718119164804e-06,
|
460 |
-
"loss": 0.
|
461 |
-
"reward": 0.
|
462 |
-
"reward_std": 0.
|
463 |
-
"rewards/accuracy_reward": 0.
|
464 |
"rewards/format_reward": 0.0,
|
465 |
"step": 170
|
466 |
},
|
467 |
{
|
468 |
-
"completion_length":
|
469 |
"epoch": 0.44333333333333336,
|
470 |
-
"grad_norm": 0.
|
471 |
-
"kl": 0.
|
472 |
"learning_rate": 2.0462408628792335e-06,
|
473 |
-
"loss": 0.
|
474 |
-
"reward": 0.
|
475 |
-
"reward_std": 0.
|
476 |
-
"rewards/accuracy_reward": 0.
|
477 |
"rewards/format_reward": 0.0,
|
478 |
"step": 175
|
479 |
},
|
480 |
{
|
481 |
-
"completion_length":
|
482 |
"epoch": 0.456,
|
483 |
-
"grad_norm": 0.
|
484 |
-
"kl": 0.
|
485 |
"learning_rate": 1.9837345729627633e-06,
|
486 |
-
"loss": 0.
|
487 |
-
"reward": 0.
|
488 |
-
"reward_std": 0.
|
489 |
-
"rewards/accuracy_reward": 0.
|
490 |
"rewards/format_reward": 0.0,
|
491 |
"step": 180
|
492 |
},
|
493 |
{
|
494 |
-
"completion_length":
|
495 |
"epoch": 0.4686666666666667,
|
496 |
-
"grad_norm": 0.
|
497 |
-
"kl": 0.
|
498 |
"learning_rate": 1.9202759933214665e-06,
|
499 |
-
"loss": 0.
|
500 |
-
"reward": 0.
|
501 |
-
"reward_std": 0.
|
502 |
-
"rewards/accuracy_reward": 0.
|
503 |
"rewards/format_reward": 0.0,
|
504 |
"step": 185
|
505 |
},
|
506 |
{
|
507 |
-
"completion_length":
|
508 |
"epoch": 0.48133333333333334,
|
509 |
-
"grad_norm": 0.
|
510 |
-
"kl": 0.
|
511 |
"learning_rate": 1.8559900498066726e-06,
|
512 |
-
"loss": 0.
|
513 |
-
"reward": 0.
|
514 |
-
"reward_std": 0.
|
515 |
-
"rewards/accuracy_reward": 0.
|
516 |
"rewards/format_reward": 0.0,
|
517 |
"step": 190
|
518 |
},
|
519 |
{
|
520 |
-
"completion_length":
|
521 |
"epoch": 0.494,
|
522 |
-
"grad_norm": 0.
|
523 |
-
"kl": 0.
|
524 |
"learning_rate": 1.7910032970350677e-06,
|
525 |
-
"loss": 0.
|
526 |
-
"reward": 0.
|
527 |
-
"reward_std": 0.
|
528 |
-
"rewards/accuracy_reward": 0.
|
529 |
"rewards/format_reward": 0.0,
|
530 |
"step": 195
|
531 |
},
|
532 |
{
|
533 |
-
"completion_length":
|
534 |
"epoch": 0.5066666666666667,
|
535 |
-
"grad_norm": 0.
|
536 |
-
"kl": 0.
|
537 |
"learning_rate": 1.7254436692507058e-06,
|
538 |
-
"loss": 0.
|
539 |
-
"reward": 0.
|
540 |
-
"reward_std": 0.
|
541 |
-
"rewards/accuracy_reward": 0.
|
542 |
"rewards/format_reward": 0.0,
|
543 |
"step": 200
|
544 |
},
|
545 |
{
|
546 |
"epoch": 0.5066666666666667,
|
547 |
-
"eval_completion_length":
|
548 |
-
"eval_kl": 0.
|
549 |
-
"eval_loss":
|
550 |
-
"eval_reward": 0.
|
551 |
-
"eval_reward_std": 0.
|
552 |
-
"eval_rewards/accuracy_reward": 0.
|
553 |
"eval_rewards/format_reward": 0.0,
|
554 |
-
"eval_runtime":
|
555 |
-
"eval_samples_per_second": 0.
|
556 |
-
"eval_steps_per_second": 0.
|
557 |
"step": 200
|
558 |
},
|
559 |
{
|
560 |
-
"completion_length":
|
561 |
"epoch": 0.5193333333333333,
|
562 |
-
"grad_norm": 0.
|
563 |
-
"kl": 0.
|
564 |
"learning_rate": 1.6594402284710481e-06,
|
565 |
-
"loss": 0.
|
566 |
-
"reward": 0.
|
567 |
-
"reward_std": 0.
|
568 |
-
"rewards/accuracy_reward": 0.
|
569 |
"rewards/format_reward": 0.0,
|
570 |
"step": 205
|
571 |
},
|
572 |
{
|
573 |
-
"completion_length":
|
574 |
"epoch": 0.532,
|
575 |
-
"grad_norm": 0.
|
576 |
-
"kl": 0.
|
577 |
"learning_rate": 1.593122910412851e-06,
|
578 |
-
"loss": 0.
|
579 |
-
"reward": 0.
|
580 |
-
"reward_std": 0.
|
581 |
-
"rewards/accuracy_reward": 0.
|
582 |
"rewards/format_reward": 0.0,
|
583 |
"step": 210
|
584 |
},
|
585 |
{
|
586 |
-
"completion_length":
|
587 |
"epoch": 0.5446666666666666,
|
588 |
-
"grad_norm":
|
589 |
-
"kl": 0.
|
590 |
"learning_rate": 1.5266222686980693e-06,
|
591 |
-
"loss": 0.
|
592 |
-
"reward": 0.
|
593 |
-
"reward_std": 0.
|
594 |
-
"rewards/accuracy_reward": 0.
|
595 |
"rewards/format_reward": 0.0,
|
596 |
"step": 215
|
597 |
},
|
598 |
{
|
599 |
-
"completion_length":
|
600 |
"epoch": 0.5573333333333333,
|
601 |
-
"grad_norm": 0.
|
602 |
-
"kl": 0.
|
603 |
"learning_rate": 1.460069217843338e-06,
|
604 |
-
"loss": 0.
|
605 |
-
"reward": 0.
|
606 |
-
"reward_std": 0.
|
607 |
-
"rewards/accuracy_reward": 0.
|
608 |
"rewards/format_reward": 0.0,
|
609 |
"step": 220
|
610 |
},
|
611 |
{
|
612 |
-
"completion_length":
|
613 |
"epoch": 0.57,
|
614 |
-
"grad_norm": 0.
|
615 |
-
"kl": 0.
|
616 |
"learning_rate": 1.3935947755389924e-06,
|
617 |
-
"loss": 0.
|
618 |
-
"reward": 0.
|
619 |
-
"reward_std": 0.
|
620 |
-
"rewards/accuracy_reward": 0.
|
621 |
"rewards/format_reward": 0.0,
|
622 |
"step": 225
|
623 |
},
|
624 |
{
|
625 |
-
"completion_length":
|
626 |
"epoch": 0.5826666666666667,
|
627 |
-
"grad_norm": 0.
|
628 |
-
"kl": 0.
|
629 |
"learning_rate": 1.3273298047249756e-06,
|
630 |
-
"loss": 0.
|
631 |
-
"reward": 0.
|
632 |
-
"reward_std": 0.
|
633 |
-
"rewards/accuracy_reward": 0.
|
634 |
"rewards/format_reward": 0.0,
|
635 |
"step": 230
|
636 |
},
|
637 |
{
|
638 |
-
"completion_length":
|
639 |
"epoch": 0.5953333333333334,
|
640 |
-
"grad_norm": 0.
|
641 |
-
"kl": 0.
|
642 |
"learning_rate": 1.2614047559713923e-06,
|
643 |
-
"loss": 0.
|
644 |
-
"reward": 0.
|
645 |
-
"reward_std": 0.
|
646 |
-
"rewards/accuracy_reward": 0.
|
647 |
"rewards/format_reward": 0.0,
|
648 |
"step": 235
|
649 |
},
|
650 |
{
|
651 |
-
"completion_length":
|
652 |
"epoch": 0.608,
|
653 |
-
"grad_norm": 0.
|
654 |
-
"kl": 0.
|
655 |
"learning_rate": 1.1959494106708598e-06,
|
656 |
-
"loss": 0.
|
657 |
-
"reward": 0.
|
658 |
-
"reward_std": 0.
|
659 |
-
"rewards/accuracy_reward": 0.
|
660 |
"rewards/format_reward": 0.0,
|
661 |
"step": 240
|
662 |
},
|
663 |
{
|
664 |
-
"completion_length":
|
665 |
"epoch": 0.6206666666666667,
|
666 |
-
"grad_norm": 0.
|
667 |
-
"kl": 0.
|
668 |
"learning_rate": 1.1310926255482204e-06,
|
669 |
-
"loss": 0.
|
670 |
-
"reward": 0.
|
671 |
-
"reward_std": 0.
|
672 |
-
"rewards/accuracy_reward": 0.
|
673 |
"rewards/format_reward": 0.0,
|
674 |
"step": 245
|
675 |
},
|
676 |
{
|
677 |
-
"completion_length":
|
678 |
"epoch": 0.6333333333333333,
|
679 |
-
"grad_norm": 0.
|
680 |
-
"kl": 0.
|
681 |
"learning_rate": 1.0669620789905688e-06,
|
682 |
-
"loss": 0.
|
683 |
-
"reward": 0.
|
684 |
-
"reward_std": 0.
|
685 |
-
"rewards/accuracy_reward": 0.
|
686 |
"rewards/format_reward": 0.0,
|
687 |
"step": 250
|
688 |
},
|
689 |
{
|
690 |
-
"completion_length":
|
691 |
"epoch": 0.646,
|
692 |
-
"grad_norm":
|
693 |
-
"kl":
|
694 |
"learning_rate": 1.0036840196969795e-06,
|
695 |
-
"loss": 0.
|
696 |
-
"reward": 0.
|
697 |
-
"reward_std": 0.
|
698 |
-
"rewards/accuracy_reward": 0.
|
699 |
"rewards/format_reward": 0.0,
|
700 |
"step": 255
|
701 |
},
|
702 |
{
|
703 |
-
"completion_length":
|
704 |
"epoch": 0.6586666666666666,
|
705 |
-
"grad_norm":
|
706 |
-
"kl":
|
707 |
"learning_rate": 9.413830181427508e-07,
|
708 |
-
"loss": 0.
|
709 |
-
"reward": 0.
|
710 |
-
"reward_std": 0.
|
711 |
-
"rewards/accuracy_reward": 0.
|
712 |
"rewards/format_reward": 0.0,
|
713 |
"step": 260
|
714 |
},
|
715 |
{
|
716 |
-
"completion_length":
|
717 |
"epoch": 0.6713333333333333,
|
718 |
-
"grad_norm": 0.
|
719 |
-
"kl":
|
720 |
"learning_rate": 8.801817213474331e-07,
|
721 |
-
"loss": 0.
|
722 |
-
"reward": 0.
|
723 |
-
"reward_std": 0.
|
724 |
-
"rewards/accuracy_reward": 0.
|
725 |
"rewards/format_reward": 0.0,
|
726 |
"step": 265
|
727 |
},
|
728 |
{
|
729 |
-
"completion_length":
|
730 |
"epoch": 0.684,
|
731 |
-
"grad_norm": 0.
|
732 |
-
"kl": 0.
|
733 |
"learning_rate": 8.202006114294044e-07,
|
734 |
-
"loss": 0.
|
735 |
-
"reward": 0.
|
736 |
-
"reward_std": 0.
|
737 |
-
"rewards/accuracy_reward": 0.
|
738 |
"rewards/format_reward": 0.0,
|
739 |
"step": 270
|
740 |
},
|
741 |
{
|
742 |
-
"completion_length":
|
743 |
"epoch": 0.6966666666666667,
|
744 |
-
"grad_norm": 0.
|
745 |
-
"kl": 0.
|
746 |
"learning_rate": 7.615577684223272e-07,
|
747 |
-
"loss": 0.
|
748 |
-
"reward": 0.
|
749 |
-
"reward_std": 0.
|
750 |
-
"rewards/accuracy_reward": 0.
|
751 |
"rewards/format_reward": 0.0,
|
752 |
"step": 275
|
753 |
},
|
754 |
{
|
755 |
-
"completion_length":
|
756 |
"epoch": 0.7093333333333334,
|
757 |
-
"grad_norm":
|
758 |
-
"kl":
|
759 |
"learning_rate": 7.043686378203864e-07,
|
760 |
-
"loss": 0.
|
761 |
-
"reward": 0.
|
762 |
-
"reward_std": 0.
|
763 |
-
"rewards/accuracy_reward": 0.
|
764 |
"rewards/format_reward": 0.0,
|
765 |
"step": 280
|
766 |
},
|
767 |
{
|
768 |
-
"completion_length":
|
769 |
"epoch": 0.722,
|
770 |
-
"grad_norm":
|
771 |
-
"kl": 0.
|
772 |
"learning_rate": 6.487458033099425e-07,
|
773 |
-
"loss": 0.
|
774 |
-
"reward": 0.
|
775 |
-
"reward_std": 0.
|
776 |
-
"rewards/accuracy_reward": 0.
|
777 |
"rewards/format_reward": 0.0,
|
778 |
"step": 285
|
779 |
},
|
780 |
{
|
781 |
-
"completion_length":
|
782 |
"epoch": 0.7346666666666667,
|
783 |
-
"grad_norm": 0.
|
784 |
-
"kl": 0.
|
785 |
"learning_rate": 5.947987651349942e-07,
|
786 |
-
"loss": 0.
|
787 |
-
"reward": 0.
|
788 |
-
"reward_std": 0.
|
789 |
-
"rewards/accuracy_reward": 0.
|
790 |
"rewards/format_reward": 0.0,
|
791 |
"step": 290
|
792 |
},
|
793 |
{
|
794 |
-
"completion_length":
|
795 |
"epoch": 0.7473333333333333,
|
796 |
-
"grad_norm":
|
797 |
-
"kl":
|
798 |
"learning_rate": 5.426337245327703e-07,
|
799 |
-
"loss": 0.
|
800 |
-
"reward": 0.
|
801 |
-
"reward_std": 0.
|
802 |
-
"rewards/accuracy_reward": 0.
|
803 |
"rewards/format_reward": 0.0,
|
804 |
"step": 295
|
805 |
},
|
806 |
{
|
807 |
-
"completion_length":
|
808 |
"epoch": 0.76,
|
809 |
-
"grad_norm":
|
810 |
-
"kl":
|
811 |
"learning_rate": 4.923533746638108e-07,
|
812 |
-
"loss": 0.
|
813 |
-
"reward": 0.
|
814 |
-
"reward_std": 0.
|
815 |
-
"rewards/accuracy_reward": 0.
|
816 |
"rewards/format_reward": 0.0,
|
817 |
"step": 300
|
818 |
},
|
819 |
{
|
820 |
"epoch": 0.76,
|
821 |
-
"eval_completion_length":
|
822 |
-
"eval_kl":
|
823 |
-
"eval_loss":
|
824 |
-
"eval_reward": 0.
|
825 |
-
"eval_reward_std": 0.
|
826 |
-
"eval_rewards/accuracy_reward": 0.
|
827 |
"eval_rewards/format_reward": 0.0,
|
828 |
-
"eval_runtime":
|
829 |
-
"eval_samples_per_second": 0.
|
830 |
-
"eval_steps_per_second": 0.
|
831 |
"step": 300
|
832 |
},
|
833 |
{
|
834 |
-
"completion_length":
|
835 |
"epoch": 0.7726666666666666,
|
836 |
-
"grad_norm": 0.
|
837 |
-
"kl":
|
838 |
"learning_rate": 4.440566984481256e-07,
|
839 |
-
"loss": 0.
|
840 |
-
"reward": 0.
|
841 |
-
"reward_std": 0.
|
842 |
-
"rewards/accuracy_reward": 0.
|
843 |
"rewards/format_reward": 0.0,
|
844 |
"step": 305
|
845 |
},
|
846 |
{
|
847 |
-
"completion_length":
|
848 |
"epoch": 0.7853333333333333,
|
849 |
-
"grad_norm": 0.
|
850 |
-
"kl": 0.
|
851 |
"learning_rate": 3.978387737053994e-07,
|
852 |
-
"loss": 0.
|
853 |
-
"reward": 0.
|
854 |
-
"reward_std": 0.
|
855 |
-
"rewards/accuracy_reward": 0.
|
856 |
"rewards/format_reward": 0.0,
|
857 |
"step": 310
|
858 |
},
|
859 |
{
|
860 |
-
"completion_length":
|
861 |
"epoch": 0.798,
|
862 |
-
"grad_norm":
|
863 |
-
"kl":
|
864 |
"learning_rate": 3.5379058598286167e-07,
|
865 |
-
"loss": 0.
|
866 |
-
"reward": 0.
|
867 |
-
"reward_std": 0.
|
868 |
-
"rewards/accuracy_reward": 0.
|
869 |
"rewards/format_reward": 0.0,
|
870 |
"step": 315
|
871 |
},
|
872 |
{
|
873 |
-
"completion_length":
|
874 |
"epoch": 0.8106666666666666,
|
875 |
-
"grad_norm":
|
876 |
-
"kl":
|
877 |
"learning_rate": 3.119988494392894e-07,
|
878 |
-
"loss": 0.
|
879 |
-
"reward": 0.
|
880 |
-
"reward_std": 0.
|
881 |
-
"rewards/accuracy_reward": 0.
|
882 |
"rewards/format_reward": 0.0,
|
883 |
"step": 320
|
884 |
},
|
885 |
{
|
886 |
-
"completion_length":
|
887 |
"epoch": 0.8233333333333334,
|
888 |
-
"grad_norm":
|
889 |
-
"kl":
|
890 |
"learning_rate": 2.725458361377465e-07,
|
891 |
-
"loss": 0.
|
892 |
-
"reward": 0.
|
893 |
-
"reward_std": 0.
|
894 |
-
"rewards/accuracy_reward": 0.
|
895 |
"rewards/format_reward": 0.0,
|
896 |
"step": 325
|
897 |
},
|
898 |
{
|
899 |
-
"completion_length":
|
900 |
"epoch": 0.836,
|
901 |
-
"grad_norm": 0.
|
902 |
-
"kl":
|
903 |
"learning_rate": 2.3550921408312737e-07,
|
904 |
-
"loss": 0.
|
905 |
-
"reward": 0.
|
906 |
-
"reward_std": 0.
|
907 |
-
"rewards/accuracy_reward": 0.
|
908 |
"rewards/format_reward": 0.0,
|
909 |
"step": 330
|
910 |
},
|
911 |
{
|
912 |
-
"completion_length":
|
913 |
"epoch": 0.8486666666666667,
|
914 |
-
"grad_norm":
|
915 |
-
"kl":
|
916 |
"learning_rate": 2.0096189432334195e-07,
|
917 |
-
"loss": 0.
|
918 |
-
"reward": 0.
|
919 |
-
"reward_std": 0.
|
920 |
-
"rewards/accuracy_reward": 0.
|
921 |
"rewards/format_reward": 0.0,
|
922 |
"step": 335
|
923 |
},
|
924 |
{
|
925 |
-
"completion_length":
|
926 |
"epoch": 0.8613333333333333,
|
927 |
-
"grad_norm": 0.
|
928 |
-
"kl":
|
929 |
"learning_rate": 1.6897188741514286e-07,
|
930 |
-
"loss": 0.
|
931 |
-
"reward": 0.
|
932 |
-
"reward_std": 0.
|
933 |
-
"rewards/accuracy_reward": 0.
|
934 |
"rewards/format_reward": 0.0,
|
935 |
"step": 340
|
936 |
},
|
937 |
{
|
938 |
-
"completion_length":
|
939 |
"epoch": 0.874,
|
940 |
-
"grad_norm": 0.
|
941 |
-
"kl":
|
942 |
"learning_rate": 1.396021695371582e-07,
|
943 |
-
"loss": 0.
|
944 |
-
"reward": 0.
|
945 |
-
"reward_std": 0.
|
946 |
-
"rewards/accuracy_reward": 0.
|
947 |
"rewards/format_reward": 0.0,
|
948 |
"step": 345
|
949 |
},
|
950 |
{
|
951 |
-
"completion_length":
|
952 |
"epoch": 0.8866666666666667,
|
953 |
-
"grad_norm": 0.
|
954 |
-
"kl":
|
955 |
"learning_rate": 1.1291055851370623e-07,
|
956 |
-
"loss": 0.
|
957 |
-
"reward": 0.
|
958 |
-
"reward_std": 0.
|
959 |
-
"rewards/accuracy_reward": 0.
|
960 |
"rewards/format_reward": 0.0,
|
961 |
"step": 350
|
962 |
},
|
963 |
{
|
964 |
-
"completion_length":
|
965 |
"epoch": 0.8993333333333333,
|
966 |
-
"grad_norm":
|
967 |
-
"kl":
|
968 |
"learning_rate": 8.894959999345015e-08,
|
969 |
-
"loss": 0.
|
970 |
-
"reward": 0.
|
971 |
-
"reward_std": 0.
|
972 |
-
"rewards/accuracy_reward": 0.
|
973 |
"rewards/format_reward": 0.0,
|
974 |
"step": 355
|
975 |
},
|
976 |
{
|
977 |
-
"completion_length":
|
978 |
"epoch": 0.912,
|
979 |
-
"grad_norm": 0.
|
980 |
-
"kl": 0.
|
981 |
"learning_rate": 6.776646400696212e-08,
|
982 |
-
"loss": 0.
|
983 |
-
"reward": 0.
|
984 |
-
"reward_std": 0.
|
985 |
-
"rewards/accuracy_reward": 0.
|
986 |
"rewards/format_reward": 0.0,
|
987 |
"step": 360
|
988 |
},
|
989 |
{
|
990 |
-
"completion_length":
|
991 |
"epoch": 0.9246666666666666,
|
992 |
-
"grad_norm":
|
993 |
-
"kl": 0.
|
994 |
"learning_rate": 4.940285210684375e-08,
|
995 |
-
"loss": 0.
|
996 |
-
"reward": 0.
|
997 |
-
"reward_std": 0.
|
998 |
-
"rewards/accuracy_reward": 0.
|
999 |
"rewards/format_reward": 0.0,
|
1000 |
"step": 365
|
1001 |
},
|
1002 |
{
|
1003 |
-
"completion_length":
|
1004 |
"epoch": 0.9373333333333334,
|
1005 |
-
"grad_norm":
|
1006 |
-
"kl":
|
1007 |
"learning_rate": 3.389491527319999e-08,
|
1008 |
-
"loss": 0.
|
1009 |
-
"reward": 0.
|
1010 |
-
"reward_std": 0.
|
1011 |
-
"rewards/accuracy_reward": 0.
|
1012 |
"rewards/format_reward": 0.0,
|
1013 |
"step": 370
|
1014 |
},
|
1015 |
{
|
1016 |
-
"completion_length":
|
1017 |
"epoch": 0.95,
|
1018 |
-
"grad_norm":
|
1019 |
-
"kl":
|
1020 |
"learning_rate": 2.127318274608381e-08,
|
1021 |
-
"loss": 0.
|
1022 |
-
"reward": 0.
|
1023 |
-
"reward_std": 0.
|
1024 |
-
"rewards/accuracy_reward": 0.
|
1025 |
"rewards/format_reward": 0.0,
|
1026 |
"step": 375
|
1027 |
},
|
1028 |
{
|
1029 |
-
"completion_length":
|
1030 |
"epoch": 0.9626666666666667,
|
1031 |
-
"grad_norm": 0.
|
1032 |
-
"kl":
|
1033 |
"learning_rate": 1.1562501925013125e-08,
|
1034 |
-
"loss": 0.
|
1035 |
-
"reward": 0.
|
1036 |
-
"reward_std": 0.
|
1037 |
-
"rewards/accuracy_reward": 0.
|
1038 |
"rewards/format_reward": 0.0,
|
1039 |
"step": 380
|
1040 |
},
|
1041 |
{
|
1042 |
-
"completion_length":
|
1043 |
"epoch": 0.9753333333333334,
|
1044 |
-
"grad_norm":
|
1045 |
-
"kl":
|
1046 |
"learning_rate": 4.781989453874814e-09,
|
1047 |
-
"loss": 0.
|
1048 |
-
"reward": 0.
|
1049 |
-
"reward_std": 0.
|
1050 |
-
"rewards/accuracy_reward": 0.
|
1051 |
"rewards/format_reward": 0.0,
|
1052 |
"step": 385
|
1053 |
},
|
1054 |
{
|
1055 |
-
"completion_length":
|
1056 |
"epoch": 0.988,
|
1057 |
-
"grad_norm":
|
1058 |
-
"kl":
|
1059 |
"learning_rate": 9.44993587509657e-10,
|
1060 |
-
"loss": 0.
|
1061 |
-
"reward": 0.
|
1062 |
-
"reward_std": 0.
|
1063 |
-
"rewards/accuracy_reward": 0.
|
1064 |
"rewards/format_reward": 0.0,
|
1065 |
"step": 390
|
1066 |
},
|
1067 |
{
|
1068 |
-
"completion_length":
|
1069 |
"epoch": 0.9981333333333333,
|
1070 |
-
"kl":
|
1071 |
-
"reward": 0.
|
1072 |
-
"reward_std": 0.
|
1073 |
-
"rewards/accuracy_reward": 0.
|
1074 |
"rewards/format_reward": 0.0,
|
1075 |
"step": 394,
|
1076 |
"total_flos": 0.0,
|
1077 |
-
"train_loss": 0.
|
1078 |
-
"train_runtime":
|
1079 |
-
"train_samples_per_second": 0.
|
1080 |
"train_steps_per_second": 0.001
|
1081 |
}
|
1082 |
],
|
|
|
9 |
"is_world_process_zero": true,
|
10 |
"log_history": [
|
11 |
{
|
12 |
+
"completion_length": 644.0193140933388,
|
13 |
"epoch": 0.012666666666666666,
|
14 |
+
"grad_norm": 0.16860076785087585,
|
15 |
+
"kl": 0.000124662016567431,
|
16 |
"learning_rate": 3.75e-07,
|
17 |
"loss": 0.0,
|
18 |
+
"reward": 0.605263171384209,
|
19 |
+
"reward_std": 0.40058013012534693,
|
20 |
+
"rewards/accuracy_reward": 0.605263171384209,
|
21 |
"rewards/format_reward": 0.0,
|
22 |
"step": 5
|
23 |
},
|
24 |
{
|
25 |
+
"completion_length": 601.6614225688734,
|
26 |
"epoch": 0.025333333333333333,
|
27 |
+
"grad_norm": 0.14201320707798004,
|
28 |
+
"kl": 0.00020345888639751233,
|
29 |
"learning_rate": 7.5e-07,
|
30 |
"loss": 0.0,
|
31 |
+
"reward": 0.673684226839166,
|
32 |
+
"reward_std": 0.39511305846666034,
|
33 |
+
"rewards/accuracy_reward": 0.673684226839166,
|
34 |
+
"rewards/format_reward": 0.0,
|
35 |
"step": 10
|
36 |
},
|
37 |
{
|
38 |
+
"completion_length": 579.6684371145149,
|
39 |
"epoch": 0.038,
|
40 |
+
"grad_norm": 0.3944288194179535,
|
41 |
+
"kl": 0.00023700312564247533,
|
42 |
"learning_rate": 1.125e-06,
|
43 |
"loss": 0.0,
|
44 |
+
"reward": 0.6631579123045269,
|
45 |
+
"reward_std": 0.3630171424464176,
|
46 |
+
"rewards/accuracy_reward": 0.6631579123045269,
|
47 |
"rewards/format_reward": 0.0,
|
48 |
"step": 15
|
49 |
},
|
50 |
{
|
51 |
+
"completion_length": 615.7789647152549,
|
52 |
"epoch": 0.050666666666666665,
|
53 |
+
"grad_norm": 0.1742940992116928,
|
54 |
+
"kl": 0.0005895815397563733,
|
55 |
"learning_rate": 1.5e-06,
|
56 |
"loss": 0.0,
|
57 |
+
"reward": 0.6175438761711121,
|
58 |
+
"reward_std": 0.40319750308990476,
|
59 |
+
"rewards/accuracy_reward": 0.6175438761711121,
|
60 |
"rewards/format_reward": 0.0,
|
61 |
"step": 20
|
62 |
},
|
63 |
{
|
64 |
+
"completion_length": 653.6140499717311,
|
65 |
"epoch": 0.06333333333333334,
|
66 |
+
"grad_norm": 0.213926300406456,
|
67 |
+
"kl": 0.0014986289174933182,
|
68 |
"learning_rate": 1.875e-06,
|
69 |
"loss": 0.0,
|
70 |
+
"reward": 0.6561403632164001,
|
71 |
+
"reward_std": 0.3638197688679946,
|
72 |
+
"rewards/accuracy_reward": 0.6561403632164001,
|
73 |
"rewards/format_reward": 0.0,
|
74 |
"step": 25
|
75 |
},
|
76 |
{
|
77 |
+
"completion_length": 624.1438784950658,
|
78 |
"epoch": 0.076,
|
79 |
+
"grad_norm": 0.16140718758106232,
|
80 |
+
"kl": 0.0023109034488075658,
|
81 |
"learning_rate": 2.25e-06,
|
82 |
+
"loss": 0.0,
|
83 |
+
"reward": 0.6947368581044047,
|
84 |
+
"reward_std": 0.32959623650500647,
|
85 |
+
"rewards/accuracy_reward": 0.6947368581044047,
|
86 |
"rewards/format_reward": 0.0,
|
87 |
"step": 30
|
88 |
},
|
89 |
{
|
90 |
+
"completion_length": 649.8438781738281,
|
91 |
"epoch": 0.08866666666666667,
|
92 |
+
"grad_norm": 0.16571274399757385,
|
93 |
+
"kl": 0.009154390033922698,
|
94 |
"learning_rate": 2.6250000000000003e-06,
|
95 |
+
"loss": 0.0,
|
96 |
+
"reward": 0.6894737009939395,
|
97 |
+
"reward_std": 0.35860375699244046,
|
98 |
+
"rewards/accuracy_reward": 0.687719314819888,
|
99 |
+
"rewards/format_reward": 0.0017543860171970569,
|
100 |
"step": 35
|
101 |
},
|
102 |
{
|
103 |
+
"completion_length": 629.4052797016344,
|
104 |
"epoch": 0.10133333333333333,
|
105 |
+
"grad_norm": 0.27635857462882996,
|
106 |
+
"kl": 0.0027755938078227794,
|
107 |
"learning_rate": 3e-06,
|
108 |
+
"loss": 0.0,
|
109 |
+
"reward": 0.7035087861512836,
|
110 |
+
"reward_std": 0.34189380752412896,
|
111 |
+
"rewards/accuracy_reward": 0.7035087861512836,
|
112 |
"rewards/format_reward": 0.0,
|
113 |
"step": 40
|
114 |
},
|
115 |
{
|
116 |
+
"completion_length": 652.133349609375,
|
117 |
"epoch": 0.114,
|
118 |
+
"grad_norm": 0.0473485067486763,
|
119 |
+
"kl": 0.026468778911389802,
|
120 |
"learning_rate": 2.998523534736735e-06,
|
121 |
+
"loss": 0.0,
|
122 |
+
"reward": 0.6842105420012223,
|
123 |
+
"reward_std": 0.3472982666994396,
|
124 |
+
"rewards/accuracy_reward": 0.6842105420012223,
|
125 |
"rewards/format_reward": 0.0,
|
126 |
"step": 45
|
127 |
},
|
128 |
{
|
129 |
+
"completion_length": 628.0421226099918,
|
130 |
"epoch": 0.12666666666666668,
|
131 |
+
"grad_norm": 0.11586566269397736,
|
132 |
+
"kl": 0.004013121755499589,
|
133 |
"learning_rate": 2.994097045546504e-06,
|
134 |
+
"loss": 0.0,
|
135 |
+
"reward": 0.7263158050022627,
|
136 |
+
"reward_std": 0.3308417533573351,
|
137 |
+
"rewards/accuracy_reward": 0.7263158050022627,
|
138 |
"rewards/format_reward": 0.0,
|
139 |
"step": 50
|
140 |
},
|
141 |
{
|
142 |
+
"completion_length": 634.6701881810238,
|
143 |
"epoch": 0.13933333333333334,
|
144 |
+
"grad_norm": 0.09287750720977783,
|
145 |
+
"kl": 0.0033212159809313324,
|
146 |
"learning_rate": 2.986729246506011e-06,
|
147 |
+
"loss": 0.0,
|
148 |
+
"reward": 0.6912280863837192,
|
149 |
+
"reward_std": 0.35596638729697777,
|
150 |
+
"rewards/accuracy_reward": 0.6912280863837192,
|
151 |
"rewards/format_reward": 0.0,
|
152 |
"step": 55
|
153 |
},
|
154 |
{
|
155 |
+
"completion_length": 607.8982656378495,
|
156 |
"epoch": 0.152,
|
157 |
+
"grad_norm": 0.1364029198884964,
|
158 |
+
"kl": 0.003622757761101974,
|
159 |
"learning_rate": 2.976434642014389e-06,
|
160 |
+
"loss": 0.0,
|
161 |
+
"reward": 0.7561403630595458,
|
162 |
+
"reward_std": 0.2851124694472865,
|
163 |
+
"rewards/accuracy_reward": 0.7543859771992031,
|
164 |
+
"rewards/format_reward": 0.0017543860171970569,
|
165 |
"step": 60
|
166 |
},
|
167 |
{
|
168 |
+
"completion_length": 605.7438781738281,
|
169 |
"epoch": 0.16466666666666666,
|
170 |
+
"grad_norm": 0.0922616571187973,
|
171 |
+
"kl": 0.007114209626850329,
|
172 |
"learning_rate": 2.9632334982395456e-06,
|
173 |
+
"loss": 0.0,
|
174 |
+
"reward": 0.7578947509589948,
|
175 |
+
"reward_std": 0.29374928286201074,
|
176 |
+
"rewards/accuracy_reward": 0.7578947509589948,
|
177 |
"rewards/format_reward": 0.0,
|
178 |
"step": 65
|
179 |
},
|
180 |
{
|
181 |
+
"completion_length": 597.1175623843544,
|
182 |
"epoch": 0.17733333333333334,
|
183 |
+
"grad_norm": 0.0,
|
184 |
+
"kl": 0.003088178132709704,
|
185 |
"learning_rate": 2.947151803221774e-06,
|
186 |
+
"loss": 0.0,
|
187 |
+
"reward": 0.7596491361919202,
|
188 |
+
"reward_std": 0.2632309951280293,
|
189 |
+
"rewards/accuracy_reward": 0.7596491361919202,
|
190 |
"rewards/format_reward": 0.0,
|
191 |
"step": 70
|
192 |
},
|
193 |
{
|
194 |
+
"completion_length": 599.9701933208265,
|
195 |
"epoch": 0.19,
|
196 |
+
"grad_norm": 0.10758549720048904,
|
197 |
+
"kl": 0.003361149838096217,
|
198 |
"learning_rate": 2.928221215713164e-06,
|
199 |
+
"loss": 0.0,
|
200 |
+
"reward": 0.7666666791627281,
|
201 |
+
"reward_std": 0.2686667611724452,
|
202 |
+
"rewards/accuracy_reward": 0.7666666791627281,
|
203 |
"rewards/format_reward": 0.0,
|
204 |
"step": 75
|
205 |
},
|
206 |
{
|
207 |
+
"completion_length": 597.1456305252879,
|
208 |
"epoch": 0.20266666666666666,
|
209 |
+
"grad_norm": 0.06053958460688591,
|
210 |
+
"kl": 0.0037446674547697368,
|
211 |
"learning_rate": 2.906479002853542e-06,
|
212 |
+
"loss": 0.0,
|
213 |
+
"reward": 0.7315789610147476,
|
214 |
+
"reward_std": 0.30245108447576824,
|
215 |
+
"rewards/accuracy_reward": 0.7315789610147476,
|
216 |
"rewards/format_reward": 0.0,
|
217 |
"step": 80
|
218 |
},
|
219 |
{
|
220 |
+
"completion_length": 577.8438778525905,
|
221 |
"epoch": 0.21533333333333332,
|
222 |
+
"grad_norm": 0.0840262621641159,
|
223 |
+
"kl": 0.004712596692537006,
|
224 |
"learning_rate": 2.8819679668056195e-06,
|
225 |
+
"loss": 0.0,
|
226 |
+
"reward": 0.7982456276291295,
|
227 |
+
"reward_std": 0.26917701614530465,
|
228 |
+
"rewards/accuracy_reward": 0.7982456276291295,
|
229 |
"rewards/format_reward": 0.0,
|
230 |
"step": 85
|
231 |
},
|
232 |
{
|
233 |
+
"completion_length": 612.2982611405222,
|
234 |
"epoch": 0.228,
|
235 |
+
"grad_norm": 0.0,
|
236 |
+
"kl": 0.02382647865696957,
|
237 |
"learning_rate": 2.8547363604937856e-06,
|
238 |
+
"loss": 0.0,
|
239 |
+
"reward": 0.7456140491523241,
|
240 |
+
"reward_std": 0.2956205848016237,
|
241 |
+
"rewards/accuracy_reward": 0.7456140491523241,
|
242 |
"rewards/format_reward": 0.0,
|
243 |
"step": 90
|
244 |
},
|
245 |
{
|
246 |
+
"completion_length": 598.1403680098684,
|
247 |
"epoch": 0.24066666666666667,
|
248 |
+
"grad_norm": 0.0,
|
249 |
+
"kl": 0.005134462055407072,
|
250 |
"learning_rate": 2.824837792612416e-06,
|
251 |
+
"loss": 0.0,
|
252 |
+
"reward": 0.7403508889047723,
|
253 |
+
"reward_std": 0.2658375608293634,
|
254 |
+
"rewards/accuracy_reward": 0.7403508889047723,
|
255 |
"rewards/format_reward": 0.0,
|
256 |
"step": 95
|
257 |
},
|
258 |
{
|
259 |
+
"completion_length": 582.6561587685032,
|
260 |
"epoch": 0.25333333333333335,
|
261 |
+
"grad_norm": 0.11761737614870071,
|
262 |
+
"kl": 0.00578155517578125,
|
263 |
"learning_rate": 2.792331122090709e-06,
|
264 |
+
"loss": 0.0,
|
265 |
+
"reward": 0.7614035217385543,
|
266 |
+
"reward_std": 0.25812496166480214,
|
267 |
+
"rewards/accuracy_reward": 0.7614035217385543,
|
268 |
"rewards/format_reward": 0.0,
|
269 |
"step": 100
|
270 |
},
|
271 |
{
|
272 |
"epoch": 0.25333333333333335,
|
273 |
+
"eval_completion_length": 576.5513494110107,
|
274 |
+
"eval_kl": 0.10147758865356446,
|
275 |
+
"eval_loss": 3.062895004291022e-08,
|
276 |
+
"eval_reward": 0.6709333465278149,
|
277 |
+
"eval_reward_std": 0.3338339452087879,
|
278 |
+
"eval_rewards/accuracy_reward": 0.6709333465278149,
|
279 |
"eval_rewards/format_reward": 0.0,
|
280 |
+
"eval_runtime": 79047.156,
|
281 |
+
"eval_samples_per_second": 0.063,
|
282 |
+
"eval_steps_per_second": 0.011,
|
283 |
"step": 100
|
284 |
},
|
285 |
{
|
286 |
+
"completion_length": 549.310542056435,
|
287 |
"epoch": 0.266,
|
288 |
+
"grad_norm": 0.08108708262443542,
|
289 |
+
"kl": 0.07660410027754934,
|
290 |
"learning_rate": 2.7572803422217976e-06,
|
291 |
+
"loss": 0.0,
|
292 |
+
"reward": 0.8140351000585054,
|
293 |
+
"reward_std": 0.2548168144728008,
|
294 |
+
"rewards/accuracy_reward": 0.8140351000585054,
|
295 |
"rewards/format_reward": 0.0,
|
296 |
"step": 105
|
297 |
},
|
298 |
{
|
299 |
+
"completion_length": 596.6351042094983,
|
300 |
"epoch": 0.2786666666666667,
|
301 |
+
"grad_norm": 0.06687445938587189,
|
302 |
+
"kl": 0.18131464907997533,
|
303 |
"learning_rate": 2.71975445468425e-06,
|
304 |
+
"loss": 0.0,
|
305 |
+
"reward": 0.7543859803362897,
|
306 |
+
"reward_std": 0.32687413692474365,
|
307 |
+
"rewards/accuracy_reward": 0.7543859803362897,
|
308 |
"rewards/format_reward": 0.0,
|
309 |
"step": 110
|
310 |
},
|
311 |
{
|
312 |
+
"completion_length": 547.8263289602179,
|
313 |
"epoch": 0.29133333333333333,
|
314 |
+
"grad_norm": 0.08061777800321579,
|
315 |
+
"kl": 4.457168900339227,
|
316 |
"learning_rate": 2.679827333703964e-06,
|
317 |
+
"loss": -0.0,
|
318 |
+
"reward": 0.7894736992685418,
|
319 |
+
"reward_std": 0.2753143636803878,
|
320 |
+
"rewards/accuracy_reward": 0.7894736992685418,
|
321 |
"rewards/format_reward": 0.0,
|
322 |
"step": 115
|
323 |
},
|
324 |
{
|
325 |
+
"completion_length": 533.67895989669,
|
326 |
"epoch": 0.304,
|
327 |
+
"grad_norm": 0.15190833806991577,
|
328 |
+
"kl": 0.005019980982730263,
|
329 |
"learning_rate": 2.637577580623858e-06,
|
330 |
+
"loss": 0.0,
|
331 |
+
"reward": 0.771929837528028,
|
332 |
+
"reward_std": 0.2711241148020092,
|
333 |
+
"rewards/accuracy_reward": 0.771929837528028,
|
334 |
"rewards/format_reward": 0.0,
|
335 |
"step": 120
|
336 |
},
|
337 |
{
|
338 |
+
"completion_length": 535.5947535464638,
|
339 |
"epoch": 0.31666666666666665,
|
340 |
+
"grad_norm": 0.24402964115142822,
|
341 |
+
"kl": 0.005212723581414474,
|
342 |
"learning_rate": 2.593088369167671e-06,
|
343 |
+
"loss": 0.0,
|
344 |
+
"reward": 0.8035087809750908,
|
345 |
+
"reward_std": 0.2135340957265151,
|
346 |
+
"rewards/accuracy_reward": 0.8035087809750908,
|
347 |
"rewards/format_reward": 0.0,
|
348 |
"step": 125
|
349 |
},
|
350 |
{
|
351 |
+
"completion_length": 532.9789643940172,
|
352 |
"epoch": 0.3293333333333333,
|
353 |
+
"grad_norm": 0.28009745478630066,
|
354 |
+
"kl": 0.005336239463404605,
|
355 |
"learning_rate": 2.5464472817024772e-06,
|
356 |
+
"loss": 0.0,
|
357 |
+
"reward": 0.8052631699725201,
|
358 |
+
"reward_std": 0.24214739736757782,
|
359 |
+
"rewards/accuracy_reward": 0.8052631699725201,
|
360 |
"rewards/format_reward": 0.0,
|
361 |
"step": 130
|
362 |
},
|
363 |
{
|
364 |
+
"completion_length": 561.8772093120374,
|
365 |
"epoch": 0.342,
|
366 |
+
"grad_norm": 0.18070097267627716,
|
367 |
+
"kl": 0.004754076505962171,
|
368 |
"learning_rate": 2.497746136822254e-06,
|
369 |
+
"loss": 0.0,
|
370 |
+
"reward": 0.7771929958933278,
|
371 |
+
"reward_std": 0.27938908150321556,
|
372 |
+
"rewards/accuracy_reward": 0.7771929958933278,
|
373 |
"rewards/format_reward": 0.0,
|
374 |
"step": 135
|
375 |
},
|
376 |
{
|
377 |
+
"completion_length": 578.6684380782278,
|
378 |
"epoch": 0.3546666666666667,
|
379 |
+
"grad_norm": 0.10906434804201126,
|
380 |
+
"kl": 0.005059493215460526,
|
381 |
"learning_rate": 2.4470808085919304e-06,
|
382 |
+
"loss": 0.0,
|
383 |
+
"reward": 0.7631579088537317,
|
384 |
+
"reward_std": 0.3001742513556229,
|
385 |
+
"rewards/accuracy_reward": 0.7631579088537317,
|
386 |
"rewards/format_reward": 0.0,
|
387 |
"step": 140
|
388 |
},
|
389 |
{
|
390 |
+
"completion_length": 602.6772094726563,
|
391 |
"epoch": 0.36733333333333335,
|
392 |
+
"grad_norm": 0.05584894120693207,
|
393 |
+
"kl": 0.004565590306332237,
|
394 |
"learning_rate": 2.3945510378077523e-06,
|
395 |
+
"loss": 0.0,
|
396 |
+
"reward": 0.7842105371387381,
|
397 |
+
"reward_std": 0.2481692125922755,
|
398 |
+
"rewards/accuracy_reward": 0.7842105371387381,
|
399 |
"rewards/format_reward": 0.0,
|
400 |
"step": 145
|
401 |
},
|
402 |
{
|
403 |
+
"completion_length": 580.9210699784128,
|
404 |
"epoch": 0.38,
|
405 |
+
"grad_norm": 0.0609310120344162,
|
406 |
+
"kl": 0.005594434236225329,
|
407 |
"learning_rate": 2.340260235645519e-06,
|
408 |
+
"loss": 0.0,
|
409 |
+
"reward": 0.7368421157723979,
|
410 |
+
"reward_std": 0.26106126277070296,
|
411 |
+
"rewards/accuracy_reward": 0.7368421157723979,
|
412 |
"rewards/format_reward": 0.0,
|
413 |
"step": 150
|
414 |
},
|
415 |
{
|
416 |
+
"completion_length": 525.4000133313631,
|
417 |
"epoch": 0.39266666666666666,
|
418 |
+
"grad_norm": 0.06953240185976028,
|
419 |
+
"kl": 0.007059599223889802,
|
420 |
"learning_rate": 2.2843152800832416e-06,
|
421 |
+
"loss": 0.0,
|
422 |
+
"reward": 0.8035087816025082,
|
423 |
+
"reward_std": 0.2169914556177039,
|
424 |
+
"rewards/accuracy_reward": 0.8035087816025082,
|
425 |
"rewards/format_reward": 0.0,
|
426 |
"step": 155
|
427 |
},
|
428 |
{
|
429 |
+
"completion_length": 559.4386140522204,
|
430 |
"epoch": 0.4053333333333333,
|
431 |
+
"grad_norm": 0.08336005359888077,
|
432 |
+
"kl": 0.007145851536800986,
|
433 |
"learning_rate": 2.2268263054989753e-06,
|
434 |
+
"loss": 0.0,
|
435 |
+
"reward": 0.7824561511215411,
|
436 |
+
"reward_std": 0.23924477821902224,
|
437 |
+
"rewards/accuracy_reward": 0.7824561511215411,
|
438 |
"rewards/format_reward": 0.0,
|
439 |
"step": 160
|
440 |
},
|
441 |
{
|
442 |
+
"completion_length": 546.5772112394634,
|
443 |
"epoch": 0.418,
|
444 |
+
"grad_norm": 0.0791773647069931,
|
445 |
+
"kl": 0.0061695299650493425,
|
446 |
"learning_rate": 2.167906485858047e-06,
|
447 |
+
"loss": 0.0,
|
448 |
+
"reward": 0.803508786308138,
|
449 |
+
"reward_std": 0.25981574560466564,
|
450 |
+
"rewards/accuracy_reward": 0.803508786308138,
|
451 |
"rewards/format_reward": 0.0,
|
452 |
"step": 165
|
453 |
},
|
454 |
{
|
455 |
+
"completion_length": 568.9280845240543,
|
456 |
"epoch": 0.43066666666666664,
|
457 |
+
"grad_norm": 0.11959535628557205,
|
458 |
+
"kl": 0.007060161389802632,
|
459 |
"learning_rate": 2.1076718119164804e-06,
|
460 |
+
"loss": 0.0,
|
461 |
+
"reward": 0.7736842216629731,
|
462 |
+
"reward_std": 0.2649867666395087,
|
463 |
+
"rewards/accuracy_reward": 0.7736842216629731,
|
464 |
"rewards/format_reward": 0.0,
|
465 |
"step": 170
|
466 |
},
|
467 |
{
|
468 |
+
"completion_length": 550.3982609799034,
|
469 |
"epoch": 0.44333333333333336,
|
470 |
+
"grad_norm": 0.0,
|
471 |
+
"kl": 0.007817157946134868,
|
472 |
"learning_rate": 2.0462408628792335e-06,
|
473 |
+
"loss": 0.0,
|
474 |
+
"reward": 0.7824561511215411,
|
475 |
+
"reward_std": 0.253275244800668,
|
476 |
+
"rewards/accuracy_reward": 0.7824561511215411,
|
477 |
"rewards/format_reward": 0.0,
|
478 |
"step": 175
|
479 |
},
|
480 |
{
|
481 |
+
"completion_length": 586.7596660413241,
|
482 |
"epoch": 0.456,
|
483 |
+
"grad_norm": 0.06831427663564682,
|
484 |
+
"kl": 0.0069172106291118425,
|
485 |
"learning_rate": 1.9837345729627633e-06,
|
486 |
+
"loss": 0.0,
|
487 |
+
"reward": 0.7438596612528751,
|
488 |
+
"reward_std": 0.2811869652647721,
|
489 |
+
"rewards/accuracy_reward": 0.7438596612528751,
|
490 |
"rewards/format_reward": 0.0,
|
491 |
"step": 180
|
492 |
},
|
493 |
{
|
494 |
+
"completion_length": 583.1228234542043,
|
495 |
"epoch": 0.4686666666666667,
|
496 |
+
"grad_norm": 0.10550453513860703,
|
497 |
+
"kl": 0.006921226099917763,
|
498 |
"learning_rate": 1.9202759933214665e-06,
|
499 |
+
"loss": 0.0,
|
500 |
+
"reward": 0.7631579093242946,
|
501 |
+
"reward_std": 0.29657848289138394,
|
502 |
+
"rewards/accuracy_reward": 0.7631579093242946,
|
503 |
"rewards/format_reward": 0.0,
|
504 |
"step": 185
|
505 |
},
|
506 |
{
|
507 |
+
"completion_length": 528.0912424187911,
|
508 |
"epoch": 0.48133333333333334,
|
509 |
+
"grad_norm": 0.0677071213722229,
|
510 |
+
"kl": 0.009208438270970395,
|
511 |
"learning_rate": 1.8559900498066726e-06,
|
512 |
+
"loss": 0.0,
|
513 |
+
"reward": 0.8140350991173795,
|
514 |
+
"reward_std": 0.22778719381282203,
|
515 |
+
"rewards/accuracy_reward": 0.8140350991173795,
|
516 |
"rewards/format_reward": 0.0,
|
517 |
"step": 190
|
518 |
},
|
519 |
{
|
520 |
+
"completion_length": 550.7807165848582,
|
521 |
"epoch": 0.494,
|
522 |
+
"grad_norm": 0.18718236684799194,
|
523 |
+
"kl": 0.008406307822779605,
|
524 |
"learning_rate": 1.7910032970350677e-06,
|
525 |
+
"loss": 0.0,
|
526 |
+
"reward": 0.7789473815968162,
|
527 |
+
"reward_std": 0.2719327967417868,
|
528 |
+
"rewards/accuracy_reward": 0.7789473815968162,
|
529 |
"rewards/format_reward": 0.0,
|
530 |
"step": 195
|
531 |
},
|
532 |
{
|
533 |
+
"completion_length": 551.1315929212068,
|
534 |
"epoch": 0.5066666666666667,
|
535 |
+
"grad_norm": 0.08260176330804825,
|
536 |
+
"kl": 0.00812426115337171,
|
537 |
"learning_rate": 1.7254436692507058e-06,
|
538 |
+
"loss": 0.0,
|
539 |
+
"reward": 0.8070175547348826,
|
540 |
+
"reward_std": 0.23249007557567797,
|
541 |
+
"rewards/accuracy_reward": 0.8070175547348826,
|
542 |
"rewards/format_reward": 0.0,
|
543 |
"step": 200
|
544 |
},
|
545 |
{
|
546 |
"epoch": 0.5066666666666667,
|
547 |
+
"eval_completion_length": 553.8582824371338,
|
548 |
+
"eval_kl": 0.02361789245605469,
|
549 |
+
"eval_loss": -4.30000654887408e-05,
|
550 |
+
"eval_reward": 0.6822333461046219,
|
551 |
+
"eval_reward_std": 0.32350232841968535,
|
552 |
+
"eval_rewards/accuracy_reward": 0.6822333461046219,
|
553 |
"eval_rewards/format_reward": 0.0,
|
554 |
+
"eval_runtime": 76484.7991,
|
555 |
+
"eval_samples_per_second": 0.065,
|
556 |
+
"eval_steps_per_second": 0.011,
|
557 |
"step": 200
|
558 |
},
|
559 |
{
|
560 |
+
"completion_length": 570.7842259457237,
|
561 |
"epoch": 0.5193333333333333,
|
562 |
+
"grad_norm": 0.14592121541500092,
|
563 |
+
"kl": 0.024847893965871712,
|
564 |
"learning_rate": 1.6594402284710481e-06,
|
565 |
+
"loss": 0.0,
|
566 |
+
"reward": 0.7561403608635853,
|
567 |
+
"reward_std": 0.24963736345893459,
|
568 |
+
"rewards/accuracy_reward": 0.7561403608635853,
|
569 |
"rewards/format_reward": 0.0,
|
570 |
"step": 205
|
571 |
},
|
572 |
{
|
573 |
+
"completion_length": 556.5035239771793,
|
574 |
"epoch": 0.532,
|
575 |
+
"grad_norm": 0.06877300888299942,
|
576 |
+
"kl": 0.011327321905838815,
|
577 |
"learning_rate": 1.593122910412851e-06,
|
578 |
+
"loss": 0.0,
|
579 |
+
"reward": 0.8210526436567307,
|
580 |
+
"reward_std": 0.22859587669372558,
|
581 |
+
"rewards/accuracy_reward": 0.8210526436567307,
|
582 |
"rewards/format_reward": 0.0,
|
583 |
"step": 210
|
584 |
},
|
585 |
{
|
586 |
+
"completion_length": 579.1263334575452,
|
587 |
"epoch": 0.5446666666666666,
|
588 |
+
"grad_norm": 0.0,
|
589 |
+
"kl": 0.007683683696546053,
|
590 |
"learning_rate": 1.5266222686980693e-06,
|
591 |
+
"loss": 0.0,
|
592 |
+
"reward": 0.7561403643143805,
|
593 |
+
"reward_std": 0.2832074833543677,
|
594 |
+
"rewards/accuracy_reward": 0.7561403643143805,
|
595 |
"rewards/format_reward": 0.0,
|
596 |
"step": 215
|
597 |
},
|
598 |
{
|
599 |
+
"completion_length": 588.4649293598376,
|
600 |
"epoch": 0.5573333333333333,
|
601 |
+
"grad_norm": 0.07735468447208405,
|
602 |
+
"kl": 0.007682238127055921,
|
603 |
"learning_rate": 1.460069217843338e-06,
|
604 |
+
"loss": 0.0,
|
605 |
+
"reward": 0.7333333465613817,
|
606 |
+
"reward_std": 0.31152473405787817,
|
607 |
+
"rewards/accuracy_reward": 0.7333333465613817,
|
608 |
"rewards/format_reward": 0.0,
|
609 |
"step": 220
|
610 |
},
|
611 |
{
|
612 |
+
"completion_length": 582.8508938438014,
|
613 |
"epoch": 0.57,
|
614 |
+
"grad_norm": 0.04747384414076805,
|
615 |
+
"kl": 0.008666349712171053,
|
616 |
"learning_rate": 1.3935947755389924e-06,
|
617 |
+
"loss": 0.0,
|
618 |
+
"reward": 0.7456140472700722,
|
619 |
+
"reward_std": 0.27623014795152767,
|
620 |
+
"rewards/accuracy_reward": 0.7456140472700722,
|
621 |
"rewards/format_reward": 0.0,
|
622 |
"step": 225
|
623 |
},
|
624 |
{
|
625 |
+
"completion_length": 541.6929962158204,
|
626 |
"epoch": 0.5826666666666667,
|
627 |
+
"grad_norm": 0.0,
|
628 |
+
"kl": 0.009119616056743421,
|
629 |
"learning_rate": 1.3273298047249756e-06,
|
630 |
+
"loss": 0.0,
|
631 |
+
"reward": 0.7964912389454089,
|
632 |
+
"reward_std": 0.2623067799367403,
|
633 |
+
"rewards/accuracy_reward": 0.7964912389454089,
|
634 |
"rewards/format_reward": 0.0,
|
635 |
"step": 230
|
636 |
},
|
637 |
{
|
638 |
+
"completion_length": 555.8666826750102,
|
639 |
"epoch": 0.5953333333333334,
|
640 |
+
"grad_norm": 0.1013207659125328,
|
641 |
+
"kl": 0.008411447625411184,
|
642 |
"learning_rate": 1.2614047559713923e-06,
|
643 |
+
"loss": 0.0,
|
644 |
+
"reward": 0.8263158042179911,
|
645 |
+
"reward_std": 0.25948601145493355,
|
646 |
+
"rewards/accuracy_reward": 0.8263158042179911,
|
647 |
"rewards/format_reward": 0.0,
|
648 |
"step": 235
|
649 |
},
|
650 |
{
|
651 |
+
"completion_length": 564.3929960552015,
|
652 |
"epoch": 0.608,
|
653 |
+
"grad_norm": 0.08883073180913925,
|
654 |
+
"kl": 0.008549579821134868,
|
655 |
"learning_rate": 1.1959494106708598e-06,
|
656 |
+
"loss": 0.0,
|
657 |
+
"reward": 0.792982468950121,
|
658 |
+
"reward_std": 0.25643417960719056,
|
659 |
+
"rewards/accuracy_reward": 0.792982468950121,
|
660 |
"rewards/format_reward": 0.0,
|
661 |
"step": 240
|
662 |
},
|
663 |
{
|
664 |
+
"completion_length": 536.9596663625617,
|
665 |
"epoch": 0.6206666666666667,
|
666 |
+
"grad_norm": 0.04451132193207741,
|
667 |
+
"kl": 0.009548789576480263,
|
668 |
"learning_rate": 1.1310926255482204e-06,
|
669 |
+
"loss": 0.0,
|
670 |
+
"reward": 0.8052631693451028,
|
671 |
+
"reward_std": 0.24199818247242977,
|
672 |
+
"rewards/accuracy_reward": 0.8052631693451028,
|
673 |
"rewards/format_reward": 0.0,
|
674 |
"step": 245
|
675 |
},
|
676 |
{
|
677 |
+
"completion_length": 544.7193134508635,
|
678 |
"epoch": 0.6333333333333333,
|
679 |
+
"grad_norm": 0.0,
|
680 |
+
"kl": 0.009332596628289473,
|
681 |
"learning_rate": 1.0669620789905688e-06,
|
682 |
+
"loss": 0.0,
|
683 |
+
"reward": 0.800000010038677,
|
684 |
+
"reward_std": 0.23219164547167326,
|
685 |
+
"rewards/accuracy_reward": 0.800000010038677,
|
686 |
"rewards/format_reward": 0.0,
|
687 |
"step": 250
|
688 |
},
|
689 |
{
|
690 |
+
"completion_length": 552.8649261474609,
|
691 |
"epoch": 0.646,
|
692 |
+
"grad_norm": 0.06963980942964554,
|
693 |
+
"kl": 0.009455148797286185,
|
694 |
"learning_rate": 1.0036840196969795e-06,
|
695 |
+
"loss": 0.0,
|
696 |
+
"reward": 0.807017553009485,
|
697 |
+
"reward_std": 0.21750171090427198,
|
698 |
+
"rewards/accuracy_reward": 0.807017553009485,
|
699 |
"rewards/format_reward": 0.0,
|
700 |
"step": 255
|
701 |
},
|
702 |
{
|
703 |
+
"completion_length": 560.9298404091282,
|
704 |
"epoch": 0.6586666666666666,
|
705 |
+
"grad_norm": 0.13915280997753143,
|
706 |
+
"kl": 0.008615754780016447,
|
707 |
"learning_rate": 9.413830181427508e-07,
|
708 |
+
"loss": 0.0,
|
709 |
+
"reward": 0.7894736975431442,
|
710 |
+
"reward_std": 0.26591097994854573,
|
711 |
+
"rewards/accuracy_reward": 0.7894736975431442,
|
712 |
"rewards/format_reward": 0.0,
|
713 |
"step": 260
|
714 |
},
|
715 |
{
|
716 |
+
"completion_length": 583.3000167043585,
|
717 |
"epoch": 0.6713333333333333,
|
718 |
+
"grad_norm": 0.0,
|
719 |
+
"kl": 0.008441322728207237,
|
720 |
"learning_rate": 8.801817213474331e-07,
|
721 |
+
"loss": 0.0,
|
722 |
+
"reward": 0.7508772039099744,
|
723 |
+
"reward_std": 0.2809643321915677,
|
724 |
+
"rewards/accuracy_reward": 0.7508772039099744,
|
725 |
"rewards/format_reward": 0.0,
|
726 |
"step": 265
|
727 |
},
|
728 |
{
|
729 |
+
"completion_length": 531.2859802246094,
|
730 |
"epoch": 0.684,
|
731 |
+
"grad_norm": 0.05701286345720291,
|
732 |
+
"kl": 0.00885218570106908,
|
733 |
"learning_rate": 8.202006114294044e-07,
|
734 |
+
"loss": 0.0,
|
735 |
+
"reward": 0.7877193099574039,
|
736 |
+
"reward_std": 0.2481692119648582,
|
737 |
+
"rewards/accuracy_reward": 0.7877193099574039,
|
738 |
"rewards/format_reward": 0.0,
|
739 |
"step": 270
|
740 |
},
|
741 |
{
|
742 |
+
"completion_length": 531.2403676886307,
|
743 |
"epoch": 0.6966666666666667,
|
744 |
+
"grad_norm": 0.05670388042926788,
|
745 |
+
"kl": 0.009579307154605263,
|
746 |
"learning_rate": 7.615577684223272e-07,
|
747 |
+
"loss": 0.0,
|
748 |
+
"reward": 0.8175438716223365,
|
749 |
+
"reward_std": 0.23557321397881759,
|
750 |
+
"rewards/accuracy_reward": 0.8175438716223365,
|
751 |
"rewards/format_reward": 0.0,
|
752 |
"step": 275
|
753 |
},
|
754 |
{
|
755 |
+
"completion_length": 544.3842264275802,
|
756 |
"epoch": 0.7093333333333334,
|
757 |
+
"grad_norm": 0.0,
|
758 |
+
"kl": 0.009712299547697369,
|
759 |
"learning_rate": 7.043686378203864e-07,
|
760 |
+
"loss": 0.0,
|
761 |
+
"reward": 0.7701754490011616,
|
762 |
+
"reward_std": 0.25272287663660553,
|
763 |
+
"rewards/accuracy_reward": 0.7701754490011616,
|
764 |
"rewards/format_reward": 0.0,
|
765 |
"step": 280
|
766 |
},
|
767 |
{
|
768 |
+
"completion_length": 571.7438746402138,
|
769 |
"epoch": 0.722,
|
770 |
+
"grad_norm": 0.08127343654632568,
|
771 |
+
"kl": 0.008696064196134869,
|
772 |
"learning_rate": 6.487458033099425e-07,
|
773 |
+
"loss": 0.0,
|
774 |
+
"reward": 0.7701754521382482,
|
775 |
+
"reward_std": 0.2902606151605907,
|
776 |
+
"rewards/accuracy_reward": 0.7701754521382482,
|
777 |
"rewards/format_reward": 0.0,
|
778 |
"step": 285
|
779 |
},
|
780 |
{
|
781 |
+
"completion_length": 532.1596609015214,
|
782 |
"epoch": 0.7346666666666667,
|
783 |
+
"grad_norm": 0.072137750685215,
|
784 |
+
"kl": 0.008551828484786184,
|
785 |
"learning_rate": 5.947987651349942e-07,
|
786 |
+
"loss": 0.0,
|
787 |
+
"reward": 0.8070175544211738,
|
788 |
+
"reward_std": 0.21213936178307785,
|
789 |
+
"rewards/accuracy_reward": 0.8070175544211738,
|
790 |
"rewards/format_reward": 0.0,
|
791 |
"step": 290
|
792 |
},
|
793 |
{
|
794 |
+
"completion_length": 535.1210667660362,
|
795 |
"epoch": 0.7473333333333333,
|
796 |
+
"grad_norm": 0.07434926927089691,
|
797 |
+
"kl": 0.008226896587171052,
|
798 |
"learning_rate": 5.426337245327703e-07,
|
799 |
+
"loss": 0.0,
|
800 |
+
"reward": 0.8157894832523246,
|
801 |
+
"reward_std": 0.22407589335190622,
|
802 |
+
"rewards/accuracy_reward": 0.8157894832523246,
|
803 |
"rewards/format_reward": 0.0,
|
804 |
"step": 295
|
805 |
},
|
806 |
{
|
807 |
+
"completion_length": 536.6631729929071,
|
808 |
"epoch": 0.76,
|
809 |
+
"grad_norm": 0.08616889268159866,
|
810 |
+
"kl": 0.010335339997944079,
|
811 |
"learning_rate": 4.923533746638108e-07,
|
812 |
+
"loss": 0.0,
|
813 |
+
"reward": 0.8105263270829853,
|
814 |
+
"reward_std": 0.24915841535518043,
|
815 |
+
"rewards/accuracy_reward": 0.8105263270829853,
|
816 |
"rewards/format_reward": 0.0,
|
817 |
"step": 300
|
818 |
},
|
819 |
{
|
820 |
"epoch": 0.76,
|
821 |
+
"eval_completion_length": 545.0558823242187,
|
822 |
+
"eval_kl": 0.6695549743652344,
|
823 |
+
"eval_loss": 3.003481197083602e-08,
|
824 |
+
"eval_reward": 0.6868333460628986,
|
825 |
+
"eval_reward_std": 0.3212484232842922,
|
826 |
+
"eval_rewards/accuracy_reward": 0.6868333460628986,
|
827 |
"eval_rewards/format_reward": 0.0,
|
828 |
+
"eval_runtime": 75622.6169,
|
829 |
+
"eval_samples_per_second": 0.066,
|
830 |
+
"eval_steps_per_second": 0.011,
|
831 |
"step": 300
|
832 |
},
|
833 |
{
|
834 |
+
"completion_length": 574.1017703407689,
|
835 |
"epoch": 0.7726666666666666,
|
836 |
+
"grad_norm": 0.0810866504907608,
|
837 |
+
"kl": 0.02553052400287829,
|
838 |
"learning_rate": 4.440566984481256e-07,
|
839 |
+
"loss": 0.0,
|
840 |
+
"reward": 0.7666666799469998,
|
841 |
+
"reward_std": 0.27946487539692927,
|
842 |
+
"rewards/accuracy_reward": 0.7666666799469998,
|
843 |
"rewards/format_reward": 0.0,
|
844 |
"step": 305
|
845 |
},
|
846 |
{
|
847 |
+
"completion_length": 550.2579098350124,
|
848 |
"epoch": 0.7853333333333333,
|
849 |
+
"grad_norm": 0.09656031429767609,
|
850 |
+
"kl": 0.009140657123766447,
|
851 |
"learning_rate": 3.978387737053994e-07,
|
852 |
+
"loss": 0.0,
|
853 |
+
"reward": 0.7824561517489584,
|
854 |
+
"reward_std": 0.23484032687387968,
|
855 |
+
"rewards/accuracy_reward": 0.7824561517489584,
|
856 |
"rewards/format_reward": 0.0,
|
857 |
"step": 310
|
858 |
},
|
859 |
{
|
860 |
+
"completion_length": 569.4912441855982,
|
861 |
"epoch": 0.798,
|
862 |
+
"grad_norm": 0.094791479408741,
|
863 |
+
"kl": 0.009346088610197369,
|
864 |
"learning_rate": 3.5379058598286167e-07,
|
865 |
+
"loss": 0.0,
|
866 |
+
"reward": 0.7666666777510392,
|
867 |
+
"reward_std": 0.25904074530852467,
|
868 |
+
"rewards/accuracy_reward": 0.7666666777510392,
|
869 |
"rewards/format_reward": 0.0,
|
870 |
"step": 315
|
871 |
},
|
872 |
{
|
873 |
+
"completion_length": 579.2614203202097,
|
874 |
"epoch": 0.8106666666666666,
|
875 |
+
"grad_norm": 0.10730049759149551,
|
876 |
+
"kl": 4.134726434004934,
|
877 |
"learning_rate": 3.119988494392894e-07,
|
878 |
+
"loss": 0.0,
|
879 |
+
"reward": 0.7508772053216634,
|
880 |
+
"reward_std": 0.2928671808619248,
|
881 |
+
"rewards/accuracy_reward": 0.7508772053216634,
|
882 |
"rewards/format_reward": 0.0,
|
883 |
"step": 320
|
884 |
},
|
885 |
{
|
886 |
+
"completion_length": 556.2263304057874,
|
887 |
"epoch": 0.8233333333333334,
|
888 |
+
"grad_norm": 0.07251843065023422,
|
889 |
+
"kl": 0.009259033203125,
|
890 |
"learning_rate": 2.725458361377465e-07,
|
891 |
+
"loss": 0.0,
|
892 |
+
"reward": 0.7736842222903905,
|
893 |
+
"reward_std": 0.24773237705230713,
|
894 |
+
"rewards/accuracy_reward": 0.7736842222903905,
|
895 |
"rewards/format_reward": 0.0,
|
896 |
"step": 325
|
897 |
},
|
898 |
{
|
899 |
+
"completion_length": 537.1491384405839,
|
900 |
"epoch": 0.836,
|
901 |
+
"grad_norm": 0.0,
|
902 |
+
"kl": 0.009849789268092105,
|
903 |
"learning_rate": 2.3550921408312737e-07,
|
904 |
+
"loss": 0.0,
|
905 |
+
"reward": 0.8140350983331078,
|
906 |
+
"reward_std": 0.23072349554614016,
|
907 |
+
"rewards/accuracy_reward": 0.8140350983331078,
|
908 |
"rewards/format_reward": 0.0,
|
909 |
"step": 330
|
910 |
},
|
911 |
{
|
912 |
+
"completion_length": 547.5140491686369,
|
913 |
"epoch": 0.8486666666666667,
|
914 |
+
"grad_norm": 0.08143898099660873,
|
915 |
+
"kl": 0.010343531558388158,
|
916 |
"learning_rate": 2.0096189432334195e-07,
|
917 |
+
"loss": 0.0,
|
918 |
+
"reward": 0.7824561493961435,
|
919 |
+
"reward_std": 0.23674531171196386,
|
920 |
+
"rewards/accuracy_reward": 0.7824561493961435,
|
921 |
"rewards/format_reward": 0.0,
|
922 |
"step": 335
|
923 |
},
|
924 |
{
|
925 |
+
"completion_length": 541.5175582082649,
|
926 |
"epoch": 0.8613333333333333,
|
927 |
+
"grad_norm": 0.15802732110023499,
|
928 |
+
"kl": 0.009666683799342105,
|
929 |
"learning_rate": 1.6897188741514286e-07,
|
930 |
+
"loss": 0.0,
|
931 |
+
"reward": 0.7666666757119329,
|
932 |
+
"reward_std": 0.24941472881718685,
|
933 |
+
"rewards/accuracy_reward": 0.7666666757119329,
|
934 |
"rewards/format_reward": 0.0,
|
935 |
"step": 340
|
936 |
},
|
937 |
{
|
938 |
+
"completion_length": 566.2824713456004,
|
939 |
"epoch": 0.874,
|
940 |
+
"grad_norm": 0.08206634968519211,
|
941 |
+
"kl": 0.009324886924342106,
|
942 |
"learning_rate": 1.396021695371582e-07,
|
943 |
+
"loss": 0.0,
|
944 |
+
"reward": 0.7543859756306598,
|
945 |
+
"reward_std": 0.23858530960584942,
|
946 |
+
"rewards/accuracy_reward": 0.7543859756306598,
|
947 |
"rewards/format_reward": 0.0,
|
948 |
"step": 345
|
949 |
},
|
950 |
{
|
951 |
+
"completion_length": 567.8473815917969,
|
952 |
"epoch": 0.8866666666666667,
|
953 |
+
"grad_norm": 0.06388476490974426,
|
954 |
+
"kl": 0.010038034539473684,
|
955 |
"learning_rate": 1.1291055851370623e-07,
|
956 |
+
"loss": 0.0,
|
957 |
+
"reward": 0.7491228202455922,
|
958 |
+
"reward_std": 0.2948876986378118,
|
959 |
+
"rewards/accuracy_reward": 0.7491228202455922,
|
960 |
"rewards/format_reward": 0.0,
|
961 |
"step": 350
|
962 |
},
|
963 |
{
|
964 |
+
"completion_length": 576.67545663934,
|
965 |
"epoch": 0.8993333333333333,
|
966 |
+
"grad_norm": 0.058260105550289154,
|
967 |
+
"kl": 0.008216456363075659,
|
968 |
"learning_rate": 8.894959999345015e-08,
|
969 |
+
"loss": 0.0,
|
970 |
+
"reward": 0.8017543960558741,
|
971 |
+
"reward_std": 0.23333006344343488,
|
972 |
+
"rewards/accuracy_reward": 0.8017543960558741,
|
973 |
"rewards/format_reward": 0.0,
|
974 |
"step": 355
|
975 |
},
|
976 |
{
|
977 |
+
"completion_length": 549.091246434262,
|
978 |
"epoch": 0.912,
|
979 |
+
"grad_norm": 0.10874364525079727,
|
980 |
+
"kl": 0.00869959781044408,
|
981 |
"learning_rate": 6.776646400696212e-08,
|
982 |
+
"loss": 0.0,
|
983 |
+
"reward": 0.796491244749019,
|
984 |
+
"reward_std": 0.3010250455454776,
|
985 |
+
"rewards/accuracy_reward": 0.796491244749019,
|
986 |
"rewards/format_reward": 0.0,
|
987 |
"step": 360
|
988 |
},
|
989 |
{
|
990 |
+
"completion_length": 560.8719421386719,
|
991 |
"epoch": 0.9246666666666666,
|
992 |
+
"grad_norm": 0.10691142827272415,
|
993 |
+
"kl": 0.009839509662828948,
|
994 |
"learning_rate": 4.940285210684375e-08,
|
995 |
+
"loss": 0.0,
|
996 |
+
"reward": 0.7964912431804757,
|
997 |
+
"reward_std": 0.28669615042837043,
|
998 |
+
"rewards/accuracy_reward": 0.7964912431804757,
|
999 |
"rewards/format_reward": 0.0,
|
1000 |
"step": 365
|
1001 |
},
|
1002 |
{
|
1003 |
+
"completion_length": 570.056158126028,
|
1004 |
"epoch": 0.9373333333333334,
|
1005 |
+
"grad_norm": 0.14333242177963257,
|
1006 |
+
"kl": 0.008819580078125,
|
1007 |
"learning_rate": 3.389491527319999e-08,
|
1008 |
+
"loss": 0.0,
|
1009 |
+
"reward": 0.7631579105791293,
|
1010 |
+
"reward_std": 0.3113442129210422,
|
1011 |
+
"rewards/accuracy_reward": 0.7631579105791293,
|
1012 |
"rewards/format_reward": 0.0,
|
1013 |
"step": 370
|
1014 |
},
|
1015 |
{
|
1016 |
+
"completion_length": 581.7122971384149,
|
1017 |
"epoch": 0.95,
|
1018 |
+
"grad_norm": 0.12897399067878723,
|
1019 |
+
"kl": 0.009299830386513159,
|
1020 |
"learning_rate": 2.127318274608381e-08,
|
1021 |
+
"loss": 0.0,
|
1022 |
+
"reward": 0.759649133211688,
|
1023 |
+
"reward_std": 0.24736052971137198,
|
1024 |
+
"rewards/accuracy_reward": 0.759649133211688,
|
1025 |
"rewards/format_reward": 0.0,
|
1026 |
"step": 375
|
1027 |
},
|
1028 |
{
|
1029 |
+
"completion_length": 519.6631728322883,
|
1030 |
"epoch": 0.9626666666666667,
|
1031 |
+
"grad_norm": 0.046151068061590195,
|
1032 |
+
"kl": 0.018807340923108554,
|
1033 |
"learning_rate": 1.1562501925013125e-08,
|
1034 |
+
"loss": 0.0,
|
1035 |
+
"reward": 0.8105263286515286,
|
1036 |
+
"reward_std": 0.24725342863484434,
|
1037 |
+
"rewards/accuracy_reward": 0.8105263286515286,
|
1038 |
"rewards/format_reward": 0.0,
|
1039 |
"step": 380
|
1040 |
},
|
1041 |
{
|
1042 |
+
"completion_length": 522.570188984118,
|
1043 |
"epoch": 0.9753333333333334,
|
1044 |
+
"grad_norm": 0.10620765388011932,
|
1045 |
+
"kl": 0.00986279939350329,
|
1046 |
"learning_rate": 4.781989453874814e-09,
|
1047 |
+
"loss": 0.0,
|
1048 |
+
"reward": 0.7964912391022632,
|
1049 |
+
"reward_std": 0.2455626440675635,
|
1050 |
+
"rewards/accuracy_reward": 0.7964912391022632,
|
1051 |
"rewards/format_reward": 0.0,
|
1052 |
"step": 385
|
1053 |
},
|
1054 |
{
|
1055 |
+
"completion_length": 536.8315922787315,
|
1056 |
"epoch": 0.988,
|
1057 |
+
"grad_norm": 0.09291122853755951,
|
1058 |
+
"kl": 0.010211502878289473,
|
1059 |
"learning_rate": 9.44993587509657e-10,
|
1060 |
+
"loss": 0.0,
|
1061 |
+
"reward": 0.8070175572445518,
|
1062 |
+
"reward_std": 0.2619433641433716,
|
1063 |
+
"rewards/accuracy_reward": 0.8070175572445518,
|
1064 |
"rewards/format_reward": 0.0,
|
1065 |
"step": 390
|
1066 |
},
|
1067 |
{
|
1068 |
+
"completion_length": 567.2982639513517,
|
1069 |
"epoch": 0.9981333333333333,
|
1070 |
+
"kl": 0.00917715775339227,
|
1071 |
+
"reward": 0.7828947472337046,
|
1072 |
+
"reward_std": 0.2575972625299504,
|
1073 |
+
"rewards/accuracy_reward": 0.7828947472337046,
|
1074 |
"rewards/format_reward": 0.0,
|
1075 |
"step": 394,
|
1076 |
"total_flos": 0.0,
|
1077 |
+
"train_loss": 0.0008256121124141636,
|
1078 |
+
"train_runtime": 419098.9196,
|
1079 |
+
"train_samples_per_second": 0.018,
|
1080 |
"train_steps_per_second": 0.001
|
1081 |
}
|
1082 |
],
|
training_args.bin
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
size 7480
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:84f3f7010102b0e305468c1cf34da3672c32048c60d1693fa81fc6817dc89fe5
|
3 |
size 7480
|