| { | |
| "best_global_step": null, | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.05714285714285714, | |
| "eval_steps": 500, | |
| "global_step": 50, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.5208333333333333, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 3128.0, | |
| "completions/mean_length": 2584.104248046875, | |
| "completions/mean_terminated_length": 1497.2608642578125, | |
| "completions/min_length": 557.0, | |
| "completions/min_terminated_length": 557.0, | |
| "epoch": 0.001142857142857143, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.26198074221611023, | |
| "kl": 0.0, | |
| "learning_rate": 0.0, | |
| "loss": -0.0022, | |
| "num_tokens": 131153.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": 0.3948305547237396, | |
| "reward_std": 0.7732391357421875, | |
| "rewards/cosine_scaled_reward/mean": -0.062009382992982864, | |
| "rewards/cosine_scaled_reward/std": 0.43048128485679626, | |
| "rewards/format_reward/mean": 0.5208333134651184, | |
| "rewards/format_reward/std": 0.504852294921875, | |
| "step": 1 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.5833333333333333, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 3280.0, | |
| "completions/mean_length": 2761.666748046875, | |
| "completions/mean_terminated_length": 1610.4000244140625, | |
| "completions/min_length": 465.0, | |
| "completions/min_terminated_length": 465.0, | |
| "epoch": 0.002285714285714286, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.2314005047082901, | |
| "kl": 0.0, | |
| "learning_rate": 2e-07, | |
| "loss": -0.0045, | |
| "num_tokens": 271243.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": 0.4077601432800293, | |
| "reward_std": 0.8425893187522888, | |
| "rewards/cosine_scaled_reward/mean": -0.003428752301260829, | |
| "rewards/cosine_scaled_reward/std": 0.4935320317745209, | |
| "rewards/format_reward/mean": 0.4166666567325592, | |
| "rewards/format_reward/std": 0.49822381138801575, | |
| "step": 2 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.875, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 2945.0, | |
| "completions/mean_length": 3343.33349609375, | |
| "completions/mean_terminated_length": 1658.666748046875, | |
| "completions/min_length": 490.0, | |
| "completions/min_terminated_length": 490.0, | |
| "epoch": 0.0034285714285714284, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.19728681445121765, | |
| "kl": 0.0006656646728515625, | |
| "learning_rate": 4e-07, | |
| "loss": 0.0095, | |
| "num_tokens": 439577.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": -0.15455231070518494, | |
| "reward_std": 0.5764515995979309, | |
| "rewards/cosine_scaled_reward/mean": -0.17141447961330414, | |
| "rewards/cosine_scaled_reward/std": 0.32203689217567444, | |
| "rewards/format_reward/mean": 0.1875, | |
| "rewards/format_reward/std": 0.3944427967071533, | |
| "step": 3 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.39583333333333337, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 3458.0, | |
| "completions/mean_length": 2226.89599609375, | |
| "completions/mean_terminated_length": 1337.7586669921875, | |
| "completions/min_length": 407.0, | |
| "completions/min_terminated_length": 407.0, | |
| "epoch": 0.004571428571428572, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.2950705587863922, | |
| "kl": 0.0006043116251627604, | |
| "learning_rate": 6e-07, | |
| "loss": -0.001, | |
| "num_tokens": 553824.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": 0.4680083394050598, | |
| "reward_std": 0.8357078433036804, | |
| "rewards/cosine_scaled_reward/mean": -0.09815327078104019, | |
| "rewards/cosine_scaled_reward/std": 0.399366170167923, | |
| "rewards/format_reward/mean": 0.6666666865348816, | |
| "rewards/format_reward/std": 0.47639307379722595, | |
| "step": 4 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.7083333333333333, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 2603.0, | |
| "completions/mean_length": 3089.104248046875, | |
| "completions/mean_terminated_length": 1887.21435546875, | |
| "completions/min_length": 909.0, | |
| "completions/min_terminated_length": 909.0, | |
| "epoch": 0.005714285714285714, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.2482743263244629, | |
| "kl": 0.000629425048828125, | |
| "learning_rate": 8e-07, | |
| "loss": 0.0028, | |
| "num_tokens": 710213.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": -0.06966459006071091, | |
| "reward_std": 0.7608852386474609, | |
| "rewards/cosine_scaled_reward/mean": -0.20167399942874908, | |
| "rewards/cosine_scaled_reward/std": 0.3204644024372101, | |
| "rewards/format_reward/mean": 0.3333333432674408, | |
| "rewards/format_reward/std": 0.47639307379722595, | |
| "step": 5 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.7916666666666666, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 3421.0, | |
| "completions/mean_length": 3119.52099609375, | |
| "completions/mean_terminated_length": 1354.5, | |
| "completions/min_length": 554.0, | |
| "completions/min_terminated_length": 554.0, | |
| "epoch": 0.006857142857142857, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.24190759658813477, | |
| "kl": 0.0006701151529947916, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0018, | |
| "num_tokens": 868686.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": 0.026811789721250534, | |
| "reward_std": 0.7506579756736755, | |
| "rewards/cosine_scaled_reward/mean": -0.1427767425775528, | |
| "rewards/cosine_scaled_reward/std": 0.3361252248287201, | |
| "rewards/format_reward/mean": 0.3125, | |
| "rewards/format_reward/std": 0.4684174358844757, | |
| "step": 6 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.5416666666666667, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 3457.0, | |
| "completions/mean_length": 3024.291748046875, | |
| "completions/mean_terminated_length": 2362.818359375, | |
| "completions/min_length": 839.0, | |
| "completions/min_terminated_length": 839.0, | |
| "epoch": 0.008, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.20822857320308685, | |
| "kl": 0.0005512237548828125, | |
| "learning_rate": 9.989038226169207e-07, | |
| "loss": -0.009, | |
| "num_tokens": 1021658.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": 0.47669005393981934, | |
| "reward_std": 0.9081848859786987, | |
| "rewards/cosine_scaled_reward/mean": -0.031290601938962936, | |
| "rewards/cosine_scaled_reward/std": 0.47983497381210327, | |
| "rewards/format_reward/mean": 0.5416666865348816, | |
| "rewards/format_reward/std": 0.5035336017608643, | |
| "step": 7 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.6041666666666667, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 3568.0, | |
| "completions/mean_length": 2791.875, | |
| "completions/mean_terminated_length": 1582.8421630859375, | |
| "completions/min_length": 327.0, | |
| "completions/min_terminated_length": 327.0, | |
| "epoch": 0.009142857142857144, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.23235374689102173, | |
| "kl": 0.0005970001220703125, | |
| "learning_rate": 9.956206309337066e-07, | |
| "loss": -0.0081, | |
| "num_tokens": 1163480.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": 0.5300650596618652, | |
| "reward_std": 0.7924127578735352, | |
| "rewards/cosine_scaled_reward/mean": 0.03719766065478325, | |
| "rewards/cosine_scaled_reward/std": 0.4377634525299072, | |
| "rewards/format_reward/mean": 0.4583333432674408, | |
| "rewards/format_reward/std": 0.5035336017608643, | |
| "step": 8 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.7083333333333333, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 3494.0, | |
| "completions/mean_length": 3142.95849609375, | |
| "completions/mean_terminated_length": 2071.857177734375, | |
| "completions/min_length": 955.0, | |
| "completions/min_terminated_length": 955.0, | |
| "epoch": 0.010285714285714285, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.21945622563362122, | |
| "kl": 0.0006663004557291666, | |
| "learning_rate": 9.901664203302124e-07, | |
| "loss": 0.002, | |
| "num_tokens": 1322934.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": 0.09029825031757355, | |
| "reward_std": 0.8250617980957031, | |
| "rewards/cosine_scaled_reward/mean": -0.1421239972114563, | |
| "rewards/cosine_scaled_reward/std": 0.3718816637992859, | |
| "rewards/format_reward/mean": 0.375, | |
| "rewards/format_reward/std": 0.48924607038497925, | |
| "step": 9 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.625, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 3440.0, | |
| "completions/mean_length": 2639.791748046875, | |
| "completions/mean_terminated_length": 1066.111083984375, | |
| "completions/min_length": 329.0, | |
| "completions/min_terminated_length": 329.0, | |
| "epoch": 0.011428571428571429, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.2775964140892029, | |
| "kl": 0.0005779266357421875, | |
| "learning_rate": 9.825677631722435e-07, | |
| "loss": -0.0111, | |
| "num_tokens": 1457768.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": 0.31791985034942627, | |
| "reward_std": 0.7219366431236267, | |
| "rewards/cosine_scaled_reward/mean": -0.03815798461437225, | |
| "rewards/cosine_scaled_reward/std": 0.4010634124279022, | |
| "rewards/format_reward/mean": 0.3958333432674408, | |
| "rewards/format_reward/std": 0.49420398473739624, | |
| "step": 10 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.8125, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 3528.0, | |
| "completions/mean_length": 3260.8125, | |
| "completions/mean_terminated_length": 1860.3333740234375, | |
| "completions/min_length": 855.0, | |
| "completions/min_terminated_length": 855.0, | |
| "epoch": 0.012571428571428572, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.21065565943717957, | |
| "kl": 0.0005486806233723959, | |
| "learning_rate": 9.728616793536587e-07, | |
| "loss": 0.0145, | |
| "num_tokens": 1623041.0, | |
| "policy_entropy_avg": 8.135416666666666, | |
| "reward": -0.1468753218650818, | |
| "reward_std": 0.909512996673584, | |
| "rewards/cosine_scaled_reward/mean": -0.18839001655578613, | |
| "rewards/cosine_scaled_reward/std": 0.377286821603775, | |
| "rewards/format_reward/mean": 0.2291666716337204, | |
| "rewards/format_reward/std": 0.4247443675994873, | |
| "step": 11 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.41666666666666663, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 3564.0, | |
| "completions/mean_length": 2480.791748046875, | |
| "completions/mean_terminated_length": 1692.7857666015625, | |
| "completions/min_length": 474.0, | |
| "completions/min_terminated_length": 474.0, | |
| "epoch": 0.013714285714285714, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.33562034368515015, | |
| "kl": 0.0005970001220703125, | |
| "learning_rate": 9.610954559391704e-07, | |
| "loss": 0.0138, | |
| "num_tokens": 1750327.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": 0.425642192363739, | |
| "reward_std": 0.823100745677948, | |
| "rewards/cosine_scaled_reward/mean": -0.08819279819726944, | |
| "rewards/cosine_scaled_reward/std": 0.44234269857406616, | |
| "rewards/format_reward/mean": 0.6041666865348816, | |
| "rewards/format_reward/std": 0.49420398473739624, | |
| "step": 12 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.5833333333333333, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 3538.0, | |
| "completions/mean_length": 2816.14599609375, | |
| "completions/mean_terminated_length": 1741.1500244140625, | |
| "completions/min_length": 452.0, | |
| "completions/min_terminated_length": 452.0, | |
| "epoch": 0.014857142857142857, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.27906712889671326, | |
| "kl": 0.0005658467610677084, | |
| "learning_rate": 9.473264167865171e-07, | |
| "loss": 0.0016, | |
| "num_tokens": 1893782.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": 0.24736499786376953, | |
| "reward_std": 0.7155070304870605, | |
| "rewards/cosine_scaled_reward/mean": -0.09444598108530045, | |
| "rewards/cosine_scaled_reward/std": 0.4492030441761017, | |
| "rewards/format_reward/mean": 0.4375, | |
| "rewards/format_reward/std": 0.5013279914855957, | |
| "step": 13 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.5833333333333333, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 3369.0, | |
| "completions/mean_length": 2769.0, | |
| "completions/mean_terminated_length": 1628.0, | |
| "completions/min_length": 555.0, | |
| "completions/min_terminated_length": 555.0, | |
| "epoch": 0.016, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.27333828806877136, | |
| "kl": 0.000553131103515625, | |
| "learning_rate": 9.316216432703916e-07, | |
| "loss": 0.0017, | |
| "num_tokens": 2034650.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": 0.1486106812953949, | |
| "reward_std": 0.8035473227500916, | |
| "rewards/cosine_scaled_reward/mean": -0.1336546093225479, | |
| "rewards/cosine_scaled_reward/std": 0.3953794538974762, | |
| "rewards/format_reward/mean": 0.4166666567325592, | |
| "rewards/format_reward/std": 0.49822381138801575, | |
| "step": 14 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.625, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 3186.0, | |
| "completions/mean_length": 2703.08349609375, | |
| "completions/mean_terminated_length": 1234.888916015625, | |
| "completions/min_length": 405.0, | |
| "completions/min_terminated_length": 405.0, | |
| "epoch": 0.017142857142857144, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.2734379768371582, | |
| "kl": 0.0005137125651041666, | |
| "learning_rate": 9.140576474687263e-07, | |
| "loss": -0.0122, | |
| "num_tokens": 2172588.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": 0.406665563583374, | |
| "reward_std": 0.3276861608028412, | |
| "rewards/cosine_scaled_reward/mean": 0.0168545451015234, | |
| "rewards/cosine_scaled_reward/std": 0.4574853479862213, | |
| "rewards/format_reward/mean": 0.375, | |
| "rewards/format_reward/std": 0.48924607038497925, | |
| "step": 15 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.9791666666666666, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 2984.0, | |
| "completions/mean_length": 3571.5, | |
| "completions/mean_terminated_length": 2984.0, | |
| "completions/min_length": 2984.0, | |
| "completions/min_terminated_length": 2984.0, | |
| "epoch": 0.018285714285714287, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.1932428628206253, | |
| "kl": 0.0006643931070963541, | |
| "learning_rate": 8.9471999940354e-07, | |
| "loss": 0.0075, | |
| "num_tokens": 2351850.0, | |
| "policy_entropy_avg": 8.135416666666666, | |
| "reward": -0.3992506265640259, | |
| "reward_std": 0.5042399168014526, | |
| "rewards/cosine_scaled_reward/mean": -0.22146178781986237, | |
| "rewards/cosine_scaled_reward/std": 0.292772501707077, | |
| "rewards/format_reward/mean": 0.0416666679084301, | |
| "rewards/format_reward/std": 0.20194092392921448, | |
| "step": 16 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.39583333333333337, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 3475.0, | |
| "completions/mean_length": 2287.416748046875, | |
| "completions/mean_terminated_length": 1437.9310302734375, | |
| "completions/min_length": 364.0, | |
| "completions/min_terminated_length": 364.0, | |
| "epoch": 0.019428571428571427, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.37555888295173645, | |
| "kl": 0.0006338755289713541, | |
| "learning_rate": 8.737029101523929e-07, | |
| "loss": -0.0011, | |
| "num_tokens": 2469536.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": 0.5107091665267944, | |
| "reward_std": 0.8238445520401001, | |
| "rewards/cosine_scaled_reward/mean": -0.04544559493660927, | |
| "rewards/cosine_scaled_reward/std": 0.45671001076698303, | |
| "rewards/format_reward/mean": 0.6041666865348816, | |
| "rewards/format_reward/std": 0.49420398473739624, | |
| "step": 17 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.6875, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 3116.0, | |
| "completions/mean_length": 2911.89599609375, | |
| "completions/mean_terminated_length": 1433.2667236328125, | |
| "completions/min_length": 608.0, | |
| "completions/min_terminated_length": 608.0, | |
| "epoch": 0.02057142857142857, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.21075770258903503, | |
| "kl": 0.0006434122721354166, | |
| "learning_rate": 8.511087728614862e-07, | |
| "loss": 0.0029, | |
| "num_tokens": 2617089.0, | |
| "policy_entropy_avg": 8.135416666666666, | |
| "reward": -0.13376453518867493, | |
| "reward_std": 0.6403241157531738, | |
| "rewards/cosine_scaled_reward/mean": -0.2234683483839035, | |
| "rewards/cosine_scaled_reward/std": 0.2743138074874878, | |
| "rewards/format_reward/mean": 0.3125, | |
| "rewards/format_reward/std": 0.4684174358844757, | |
| "step": 18 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.5625, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 3400.0, | |
| "completions/mean_length": 2844.5, | |
| "completions/mean_terminated_length": 1893.71435546875, | |
| "completions/min_length": 504.0, | |
| "completions/min_terminated_length": 504.0, | |
| "epoch": 0.021714285714285714, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.24317172169685364, | |
| "kl": 0.0006122589111328125, | |
| "learning_rate": 8.270476638965461e-07, | |
| "loss": -0.0105, | |
| "num_tokens": 2762067.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": 0.7856847643852234, | |
| "reward_std": 0.5978894829750061, | |
| "rewards/cosine_scaled_reward/mean": 0.15523308515548706, | |
| "rewards/cosine_scaled_reward/std": 0.5373290181159973, | |
| "rewards/format_reward/mean": 0.4791666567325592, | |
| "rewards/format_reward/std": 0.5048523545265198, | |
| "step": 19 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.47916666666666663, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 3494.0, | |
| "completions/mean_length": 2482.83349609375, | |
| "completions/mean_terminated_length": 1469.760009765625, | |
| "completions/min_length": 408.0, | |
| "completions/min_terminated_length": 408.0, | |
| "epoch": 0.022857142857142857, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.26188376545906067, | |
| "kl": 0.0005286534627278646, | |
| "learning_rate": 8.01636806561836e-07, | |
| "loss": -0.0042, | |
| "num_tokens": 2889757.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": 0.5357545614242554, | |
| "reward_std": 0.7750095129013062, | |
| "rewards/cosine_scaled_reward/mean": -0.03285994753241539, | |
| "rewards/cosine_scaled_reward/std": 0.4009867310523987, | |
| "rewards/format_reward/mean": 0.6041666865348816, | |
| "rewards/format_reward/std": 0.49420398473739624, | |
| "step": 20 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.6041666666666667, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 2710.0, | |
| "completions/mean_length": 2631.70849609375, | |
| "completions/mean_terminated_length": 1178.2105712890625, | |
| "completions/min_length": 342.0, | |
| "completions/min_terminated_length": 342.0, | |
| "epoch": 0.024, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.31193122267723083, | |
| "kl": 0.0006847381591796875, | |
| "learning_rate": 7.75e-07, | |
| "loss": 0.0002, | |
| "num_tokens": 3024185.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": 0.18238481879234314, | |
| "reward_std": 0.4078831374645233, | |
| "rewards/cosine_scaled_reward/mean": -0.11668267101049423, | |
| "rewards/cosine_scaled_reward/std": 0.3962862193584442, | |
| "rewards/format_reward/mean": 0.4166666567325592, | |
| "rewards/format_reward/std": 0.49822381138801575, | |
| "step": 21 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.27083333333333337, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 3239.0, | |
| "completions/mean_length": 1697.2083740234375, | |
| "completions/mean_terminated_length": 996.4000244140625, | |
| "completions/min_length": 250.0, | |
| "completions/min_terminated_length": 250.0, | |
| "epoch": 0.025142857142857144, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.40179967880249023, | |
| "kl": 0.0006052652994791666, | |
| "learning_rate": 7.472670160550848e-07, | |
| "loss": -0.005, | |
| "num_tokens": 3112413.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": 0.6634411811828613, | |
| "reward_std": 0.5782728791236877, | |
| "rewards/cosine_scaled_reward/mean": -0.06244581937789917, | |
| "rewards/cosine_scaled_reward/std": 0.4282727539539337, | |
| "rewards/format_reward/mean": 0.7916666865348816, | |
| "rewards/format_reward/std": 0.41041406989097595, | |
| "step": 22 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.39583333333333337, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 3421.0, | |
| "completions/mean_length": 2181.104248046875, | |
| "completions/mean_terminated_length": 1261.9654541015625, | |
| "completions/min_length": 595.0, | |
| "completions/min_terminated_length": 595.0, | |
| "epoch": 0.026285714285714287, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.3063512444496155, | |
| "kl": 0.0006097157796223959, | |
| "learning_rate": 7.185729670371604e-07, | |
| "loss": 0.0002, | |
| "num_tokens": 3225200.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": 0.3287263512611389, | |
| "reward_std": 0.8908068537712097, | |
| "rewards/cosine_scaled_reward/mean": -0.14731089770793915, | |
| "rewards/cosine_scaled_reward/std": 0.42148637771606445, | |
| "rewards/format_reward/mean": 0.625, | |
| "rewards/format_reward/std": 0.48924607038497925, | |
| "step": 23 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.5416666666666667, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 3322.0, | |
| "completions/mean_length": 2681.229248046875, | |
| "completions/mean_terminated_length": 1614.3182373046875, | |
| "completions/min_length": 461.0, | |
| "completions/min_terminated_length": 461.0, | |
| "epoch": 0.027428571428571427, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.26260870695114136, | |
| "kl": 0.0006230672200520834, | |
| "learning_rate": 6.890576474687263e-07, | |
| "loss": 0.001, | |
| "num_tokens": 3362095.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": 0.3754323720932007, | |
| "reward_std": 0.7894452810287476, | |
| "rewards/cosine_scaled_reward/mean": -0.061340540647506714, | |
| "rewards/cosine_scaled_reward/std": 0.4359513223171234, | |
| "rewards/format_reward/mean": 0.5, | |
| "rewards/format_reward/std": 0.5052911639213562, | |
| "step": 24 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.5625, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 3461.0, | |
| "completions/mean_length": 2590.166748046875, | |
| "completions/mean_terminated_length": 1312.3809814453125, | |
| "completions/min_length": 532.0, | |
| "completions/min_terminated_length": 532.0, | |
| "epoch": 0.02857142857142857, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.23011414706707, | |
| "kl": 0.0007470448811848959, | |
| "learning_rate": 6.588648530198504e-07, | |
| "loss": 0.0021, | |
| "num_tokens": 3494145.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": 0.3740345239639282, | |
| "reward_std": 0.7695837020874023, | |
| "rewards/cosine_scaled_reward/mean": -0.03079296462237835, | |
| "rewards/cosine_scaled_reward/std": 0.44012707471847534, | |
| "rewards/format_reward/mean": 0.4375, | |
| "rewards/format_reward/std": 0.5013279914855957, | |
| "step": 25 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.5625, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 3395.0, | |
| "completions/mean_length": 2929.666748046875, | |
| "completions/mean_terminated_length": 2088.381103515625, | |
| "completions/min_length": 879.0, | |
| "completions/min_terminated_length": 879.0, | |
| "epoch": 0.029714285714285714, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.23103339970111847, | |
| "kl": 0.0006039937337239584, | |
| "learning_rate": 6.281416799501187e-07, | |
| "loss": 0.0037, | |
| "num_tokens": 3642743.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": 0.2619829773902893, | |
| "reward_std": 0.6574144959449768, | |
| "rewards/cosine_scaled_reward/mean": -0.10793358087539673, | |
| "rewards/cosine_scaled_reward/std": 0.4338338077068329, | |
| "rewards/format_reward/mean": 0.4791666567325592, | |
| "rewards/format_reward/std": 0.5048523545265198, | |
| "step": 26 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.6666666666666667, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 3288.0, | |
| "completions/mean_length": 2908.20849609375, | |
| "completions/mean_terminated_length": 1556.625, | |
| "completions/min_length": 518.0, | |
| "completions/min_terminated_length": 518.0, | |
| "epoch": 0.030857142857142857, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.2881726324558258, | |
| "kl": 0.000667572021484375, | |
| "learning_rate": 5.97037808470444e-07, | |
| "loss": -0.0, | |
| "num_tokens": 3790053.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": 0.041869934648275375, | |
| "reward_std": 0.7798717021942139, | |
| "rewards/cosine_scaled_reward/mean": -0.1560431718826294, | |
| "rewards/cosine_scaled_reward/std": 0.29862359166145325, | |
| "rewards/format_reward/mean": 0.3541666567325592, | |
| "rewards/format_reward/std": 0.4833211302757263, | |
| "step": 27 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.6041666666666667, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 2977.0, | |
| "completions/mean_length": 2831.52099609375, | |
| "completions/mean_terminated_length": 1683.0, | |
| "completions/min_length": 509.0, | |
| "completions/min_terminated_length": 509.0, | |
| "epoch": 0.032, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.24444623291492462, | |
| "kl": 0.0005861918131510416, | |
| "learning_rate": 5.657047735161255e-07, | |
| "loss": 0.0062, | |
| "num_tokens": 3933718.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": 0.4484003484249115, | |
| "reward_std": 0.8719537258148193, | |
| "rewards/cosine_scaled_reward/mean": 0.006576786283403635, | |
| "rewards/cosine_scaled_reward/std": 0.4855944514274597, | |
| "rewards/format_reward/mean": 0.4375, | |
| "rewards/format_reward/std": 0.5013279914855957, | |
| "step": 28 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.7916666666666666, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 3091.0, | |
| "completions/mean_length": 3182.02099609375, | |
| "completions/mean_terminated_length": 1654.5, | |
| "completions/min_length": 418.0, | |
| "completions/min_terminated_length": 418.0, | |
| "epoch": 0.03314285714285714, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.22512345016002655, | |
| "kl": 0.0006993611653645834, | |
| "learning_rate": 5.342952264838747e-07, | |
| "loss": 0.0099, | |
| "num_tokens": 4094309.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": -0.1955069899559021, | |
| "reward_std": 0.668115496635437, | |
| "rewards/cosine_scaled_reward/mean": -0.21282805502414703, | |
| "rewards/cosine_scaled_reward/std": 0.37752190232276917, | |
| "rewards/format_reward/mean": 0.2291666716337204, | |
| "rewards/format_reward/std": 0.4247443675994873, | |
| "step": 29 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.5625, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 3320.0, | |
| "completions/mean_length": 2794.666748046875, | |
| "completions/mean_terminated_length": 1779.8095703125, | |
| "completions/min_length": 667.0, | |
| "completions/min_terminated_length": 667.0, | |
| "epoch": 0.03428571428571429, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.22448024153709412, | |
| "kl": 0.0006434122721354166, | |
| "learning_rate": 5.02962191529556e-07, | |
| "loss": 0.0021, | |
| "num_tokens": 4236355.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": 0.38854461908340454, | |
| "reward_std": 0.8984581828117371, | |
| "rewards/cosine_scaled_reward/mean": -0.0443347692489624, | |
| "rewards/cosine_scaled_reward/std": 0.44917213916778564, | |
| "rewards/format_reward/mean": 0.4791666567325592, | |
| "rewards/format_reward/std": 0.5048523545265198, | |
| "step": 30 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.7708333333333334, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 3570.0, | |
| "completions/mean_length": 3039.39599609375, | |
| "completions/mean_terminated_length": 1207.5455322265625, | |
| "completions/min_length": 281.0, | |
| "completions/min_terminated_length": 281.0, | |
| "epoch": 0.03542857142857143, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.22645282745361328, | |
| "kl": 0.0006421407063802084, | |
| "learning_rate": 4.7185832004988133e-07, | |
| "loss": 0.009, | |
| "num_tokens": 4390118.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": -0.09765049070119858, | |
| "reward_std": 0.6962664127349854, | |
| "rewards/cosine_scaled_reward/mean": -0.1740705966949463, | |
| "rewards/cosine_scaled_reward/std": 0.4055609405040741, | |
| "rewards/format_reward/mean": 0.25, | |
| "rewards/format_reward/std": 0.4375949800014496, | |
| "step": 31 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.5833333333333333, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 3517.0, | |
| "completions/mean_length": 3097.125, | |
| "completions/mean_terminated_length": 2415.5, | |
| "completions/min_length": 1046.0, | |
| "completions/min_terminated_length": 1046.0, | |
| "epoch": 0.036571428571428574, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.19842347502708435, | |
| "kl": 0.000629425048828125, | |
| "learning_rate": 4.4113514698014953e-07, | |
| "loss": -0.0137, | |
| "num_tokens": 4546544.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": 0.6700344085693359, | |
| "reward_std": 0.7424625158309937, | |
| "rewards/cosine_scaled_reward/mean": 0.10753399133682251, | |
| "rewards/cosine_scaled_reward/std": 0.5346410274505615, | |
| "rewards/format_reward/mean": 0.4583333432674408, | |
| "rewards/format_reward/std": 0.5035336017608643, | |
| "step": 32 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.75, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 3484.0, | |
| "completions/mean_length": 3236.604248046875, | |
| "completions/mean_terminated_length": 2194.416748046875, | |
| "completions/min_length": 1039.0, | |
| "completions/min_terminated_length": 1039.0, | |
| "epoch": 0.037714285714285714, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.19302453100681305, | |
| "kl": 0.0005734761555989584, | |
| "learning_rate": 4.1094235253127374e-07, | |
| "loss": 0.0048, | |
| "num_tokens": 4710313.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": -0.12012770771980286, | |
| "reward_std": 0.6003495454788208, | |
| "rewards/cosine_scaled_reward/mean": -0.1957823485136032, | |
| "rewards/cosine_scaled_reward/std": 0.28730008006095886, | |
| "rewards/format_reward/mean": 0.2708333432674408, | |
| "rewards/format_reward/std": 0.4490928649902344, | |
| "step": 33 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.45833333333333337, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 3326.0, | |
| "completions/mean_length": 2303.45849609375, | |
| "completions/mean_terminated_length": 1219.923095703125, | |
| "completions/min_length": 430.0, | |
| "completions/min_terminated_length": 430.0, | |
| "epoch": 0.038857142857142854, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.273879736661911, | |
| "kl": 0.0007279713948567709, | |
| "learning_rate": 3.8142703296283953e-07, | |
| "loss": 0.0012, | |
| "num_tokens": 4828043.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": 0.5595396757125854, | |
| "reward_std": 0.9288837313652039, | |
| "rewards/cosine_scaled_reward/mean": -7.428725803038105e-05, | |
| "rewards/cosine_scaled_reward/std": 0.5000401139259338, | |
| "rewards/format_reward/mean": 0.5625, | |
| "rewards/format_reward/std": 0.5013279914855957, | |
| "step": 34 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.7291666666666667, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 3498.0, | |
| "completions/mean_length": 3022.52099609375, | |
| "completions/mean_terminated_length": 1510.84619140625, | |
| "completions/min_length": 417.0, | |
| "completions/min_terminated_length": 417.0, | |
| "epoch": 0.04, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.25011301040649414, | |
| "kl": 0.0006783803304036459, | |
| "learning_rate": 3.5273298394491515e-07, | |
| "loss": 0.0054, | |
| "num_tokens": 4981746.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": 0.1530809849500656, | |
| "reward_std": 0.9632071256637573, | |
| "rewards/cosine_scaled_reward/mean": -0.07932490855455399, | |
| "rewards/cosine_scaled_reward/std": 0.4641749858856201, | |
| "rewards/format_reward/mean": 0.3125, | |
| "rewards/format_reward/std": 0.4684174358844757, | |
| "step": 35 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.8125, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 3483.0, | |
| "completions/mean_length": 3256.479248046875, | |
| "completions/mean_terminated_length": 1837.2222900390625, | |
| "completions/min_length": 1007.0, | |
| "completions/min_terminated_length": 1007.0, | |
| "epoch": 0.04114285714285714, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.21851785480976105, | |
| "kl": 0.0007712046305338541, | |
| "learning_rate": 3.250000000000001e-07, | |
| "loss": 0.0035, | |
| "num_tokens": 5146391.0, | |
| "policy_entropy_avg": 8.135416666666666, | |
| "reward": -0.2397136390209198, | |
| "reward_std": 0.3913343846797943, | |
| "rewards/cosine_scaled_reward/mean": -0.23504245281219482, | |
| "rewards/cosine_scaled_reward/std": 0.17867261171340942, | |
| "rewards/format_reward/mean": 0.2291666716337204, | |
| "rewards/format_reward/std": 0.4247443675994873, | |
| "step": 36 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.7708333333333334, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 3151.0, | |
| "completions/mean_length": 3157.70849609375, | |
| "completions/mean_terminated_length": 1723.8182373046875, | |
| "completions/min_length": 749.0, | |
| "completions/min_terminated_length": 749.0, | |
| "epoch": 0.04228571428571429, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.2363603562116623, | |
| "kl": 0.0006186167399088541, | |
| "learning_rate": 2.9836319343816397e-07, | |
| "loss": 0.0063, | |
| "num_tokens": 5306229.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": -0.23191750049591064, | |
| "reward_std": 0.505261242389679, | |
| "rewards/cosine_scaled_reward/mean": -0.24154144525527954, | |
| "rewards/cosine_scaled_reward/std": 0.23630201816558838, | |
| "rewards/format_reward/mean": 0.25, | |
| "rewards/format_reward/std": 0.4375949800014496, | |
| "step": 37 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.7916666666666666, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 2515.0, | |
| "completions/mean_length": 3111.854248046875, | |
| "completions/mean_terminated_length": 1317.7000732421875, | |
| "completions/min_length": 679.0, | |
| "completions/min_terminated_length": 679.0, | |
| "epoch": 0.04342857142857143, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.19270718097686768, | |
| "kl": 0.0006771087646484375, | |
| "learning_rate": 2.729523361034538e-07, | |
| "loss": -0.0004, | |
| "num_tokens": 5464382.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": 0.06942842155694962, | |
| "reward_std": 0.4463602900505066, | |
| "rewards/cosine_scaled_reward/mean": -0.07969469577074051, | |
| "rewards/cosine_scaled_reward/std": 0.3691597282886505, | |
| "rewards/format_reward/mean": 0.2291666716337204, | |
| "rewards/format_reward/std": 0.4247443675994873, | |
| "step": 38 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.5833333333333333, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 3387.0, | |
| "completions/mean_length": 2799.166748046875, | |
| "completions/mean_terminated_length": 1700.4000244140625, | |
| "completions/min_length": 262.0, | |
| "completions/min_terminated_length": 262.0, | |
| "epoch": 0.044571428571428574, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.2799771726131439, | |
| "kl": 0.0005861918131510416, | |
| "learning_rate": 2.488912271385139e-07, | |
| "loss": -0.0356, | |
| "num_tokens": 5606830.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": 0.41357025504112244, | |
| "reward_std": 0.3624642491340637, | |
| "rewards/cosine_scaled_reward/mean": -0.04217575863003731, | |
| "rewards/cosine_scaled_reward/std": 0.4245593845844269, | |
| "rewards/format_reward/mean": 0.5, | |
| "rewards/format_reward/std": 0.5052911639213562, | |
| "step": 39 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.5416666666666667, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 2945.0, | |
| "completions/mean_length": 2401.95849609375, | |
| "completions/mean_terminated_length": 1005.0, | |
| "completions/min_length": 494.0, | |
| "completions/min_terminated_length": 494.0, | |
| "epoch": 0.045714285714285714, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.2853446900844574, | |
| "kl": 0.000675201416015625, | |
| "learning_rate": 2.2629708984760706e-07, | |
| "loss": -0.0033, | |
| "num_tokens": 5729678.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": 0.2566830515861511, | |
| "reward_std": 0.573390781879425, | |
| "rewards/cosine_scaled_reward/mean": -0.11059689521789551, | |
| "rewards/cosine_scaled_reward/std": 0.43331247568130493, | |
| "rewards/format_reward/mean": 0.4791666567325592, | |
| "rewards/format_reward/std": 0.5048523545265198, | |
| "step": 40 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.625, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 3370.0, | |
| "completions/mean_length": 2863.45849609375, | |
| "completions/mean_terminated_length": 1662.5555419921875, | |
| "completions/min_length": 762.0, | |
| "completions/min_terminated_length": 762.0, | |
| "epoch": 0.046857142857142854, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.22030523419380188, | |
| "kl": 0.0006434122721354166, | |
| "learning_rate": 2.0528000059645995e-07, | |
| "loss": 0.0144, | |
| "num_tokens": 5875488.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": 0.050709377974271774, | |
| "reward_std": 0.7291332483291626, | |
| "rewards/cosine_scaled_reward/mean": -0.18285124003887177, | |
| "rewards/cosine_scaled_reward/std": 0.3616393804550171, | |
| "rewards/format_reward/mean": 0.4166666567325592, | |
| "rewards/format_reward/std": 0.49822381138801575, | |
| "step": 41 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.6875, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 2370.0, | |
| "completions/mean_length": 2728.5, | |
| "completions/mean_terminated_length": 846.4000244140625, | |
| "completions/min_length": 207.0, | |
| "completions/min_terminated_length": 207.0, | |
| "epoch": 0.048, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.3805939257144928, | |
| "kl": 0.0007483164469401041, | |
| "learning_rate": 1.8594235253127372e-07, | |
| "loss": 0.001, | |
| "num_tokens": 6014226.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": -0.11857573688030243, | |
| "reward_std": 0.35950538516044617, | |
| "rewards/cosine_scaled_reward/mean": -0.2158358097076416, | |
| "rewards/cosine_scaled_reward/std": 0.18257829546928406, | |
| "rewards/format_reward/mean": 0.3125, | |
| "rewards/format_reward/std": 0.4684174358844757, | |
| "step": 42 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.75, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 2668.0, | |
| "completions/mean_length": 3001.479248046875, | |
| "completions/mean_terminated_length": 1253.916748046875, | |
| "completions/min_length": 529.0, | |
| "completions/min_terminated_length": 529.0, | |
| "epoch": 0.04914285714285714, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.2410728931427002, | |
| "kl": 0.0007006327311197916, | |
| "learning_rate": 1.6837835672960831e-07, | |
| "loss": 0.0025, | |
| "num_tokens": 6167009.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": 0.13230201601982117, | |
| "reward_std": 0.6667492389678955, | |
| "rewards/cosine_scaled_reward/mean": -0.05851660296320915, | |
| "rewards/cosine_scaled_reward/std": 0.43021252751350403, | |
| "rewards/format_reward/mean": 0.25, | |
| "rewards/format_reward/std": 0.4375949800014496, | |
| "step": 43 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.6041666666666667, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 3552.0, | |
| "completions/mean_length": 2645.104248046875, | |
| "completions/mean_terminated_length": 1212.0526123046875, | |
| "completions/min_length": 395.0, | |
| "completions/min_terminated_length": 395.0, | |
| "epoch": 0.05028571428571429, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.2952722907066345, | |
| "kl": 0.0007654825846354166, | |
| "learning_rate": 1.5267358321348285e-07, | |
| "loss": 0.0014, | |
| "num_tokens": 6301996.0, | |
| "policy_entropy_avg": 8.135416666666666, | |
| "reward": 0.43852299451828003, | |
| "reward_std": 0.8475234508514404, | |
| "rewards/cosine_scaled_reward/mean": 0.0016132990131154656, | |
| "rewards/cosine_scaled_reward/std": 0.5085917711257935, | |
| "rewards/format_reward/mean": 0.4375, | |
| "rewards/format_reward/std": 0.5013279914855957, | |
| "step": 44 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.8541666666666666, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 3505.0, | |
| "completions/mean_length": 3466.39599609375, | |
| "completions/mean_terminated_length": 2777.571533203125, | |
| "completions/min_length": 1678.0, | |
| "completions/min_terminated_length": 1678.0, | |
| "epoch": 0.05142857142857143, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.19519105553627014, | |
| "kl": 0.0006815592447916666, | |
| "learning_rate": 1.3890454406082956e-07, | |
| "loss": 0.0045, | |
| "num_tokens": 6477125.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": 0.16751746833324432, | |
| "reward_std": 0.5252600312232971, | |
| "rewards/cosine_scaled_reward/mean": -0.030403709039092064, | |
| "rewards/cosine_scaled_reward/std": 0.44781333208084106, | |
| "rewards/format_reward/mean": 0.2291666716337204, | |
| "rewards/format_reward/std": 0.4247443675994873, | |
| "step": 45 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.7916666666666666, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 3388.0, | |
| "completions/mean_length": 3097.77099609375, | |
| "completions/mean_terminated_length": 1250.0999755859375, | |
| "completions/min_length": 605.0, | |
| "completions/min_terminated_length": 605.0, | |
| "epoch": 0.052571428571428575, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.2662387490272522, | |
| "kl": 0.0007890065511067709, | |
| "learning_rate": 1.2713832064634125e-07, | |
| "loss": 0.006, | |
| "num_tokens": 6634194.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": -0.2356199026107788, | |
| "reward_std": 0.4806956648826599, | |
| "rewards/cosine_scaled_reward/mean": -0.22256861627101898, | |
| "rewards/cosine_scaled_reward/std": 0.2471582442522049, | |
| "rewards/format_reward/mean": 0.2083333283662796, | |
| "rewards/format_reward/std": 0.41041409969329834, | |
| "step": 46 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.47916666666666663, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 3500.0, | |
| "completions/mean_length": 2685.25, | |
| "completions/mean_terminated_length": 1858.39990234375, | |
| "completions/min_length": 431.0, | |
| "completions/min_terminated_length": 431.0, | |
| "epoch": 0.053714285714285714, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.30878785252571106, | |
| "kl": 0.0005480448404947916, | |
| "learning_rate": 1.1743223682775649e-07, | |
| "loss": 0.0002, | |
| "num_tokens": 6770886.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": 0.628765344619751, | |
| "reward_std": 0.8911368250846863, | |
| "rewards/cosine_scaled_reward/mean": 0.04512912034988403, | |
| "rewards/cosine_scaled_reward/std": 0.5223999619483948, | |
| "rewards/format_reward/mean": 0.5416666865348816, | |
| "rewards/format_reward/std": 0.503533661365509, | |
| "step": 47 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.6875, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 2532.0, | |
| "completions/mean_length": 2819.6875, | |
| "completions/mean_terminated_length": 1138.2000732421875, | |
| "completions/min_length": 705.0, | |
| "completions/min_terminated_length": 705.0, | |
| "epoch": 0.054857142857142854, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.25418996810913086, | |
| "kl": 0.0007025400797526041, | |
| "learning_rate": 1.0983357966978745e-07, | |
| "loss": 0.0033, | |
| "num_tokens": 6914139.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": 0.09342099726200104, | |
| "reward_std": 0.7818130850791931, | |
| "rewards/cosine_scaled_reward/mean": -0.11972144991159439, | |
| "rewards/cosine_scaled_reward/std": 0.401507169008255, | |
| "rewards/format_reward/mean": 0.3333333432674408, | |
| "rewards/format_reward/std": 0.47639307379722595, | |
| "step": 48 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.47916666666666663, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 3209.0, | |
| "completions/mean_length": 2395.6875, | |
| "completions/mean_terminated_length": 1302.43994140625, | |
| "completions/min_length": 326.0, | |
| "completions/min_terminated_length": 326.0, | |
| "epoch": 0.056, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.2804883122444153, | |
| "kl": 0.0006554921468098959, | |
| "learning_rate": 1.0437936906629334e-07, | |
| "loss": -0.0017, | |
| "num_tokens": 7036680.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": 0.4345873296260834, | |
| "reward_std": 0.7855587005615234, | |
| "rewards/cosine_scaled_reward/mean": -0.06286442279815674, | |
| "rewards/cosine_scaled_reward/std": 0.4665209949016571, | |
| "rewards/format_reward/mean": 0.5625, | |
| "rewards/format_reward/std": 0.5013279914855957, | |
| "step": 49 | |
| }, | |
| { | |
| "clip_ratio/high_max": 0.0, | |
| "clip_ratio/high_mean": 0.0, | |
| "clip_ratio/low_mean": 0.0, | |
| "clip_ratio/low_min": 0.0, | |
| "clip_ratio/region_mean": 0.0, | |
| "completions/clipped_ratio": 0.6666666666666667, | |
| "completions/max_length": 3584.0, | |
| "completions/max_terminated_length": 2765.0, | |
| "completions/mean_length": 2816.8125, | |
| "completions/mean_terminated_length": 1282.4375, | |
| "completions/min_length": 370.0, | |
| "completions/min_terminated_length": 370.0, | |
| "epoch": 0.05714285714285714, | |
| "frac_reward_zero_std": 0.0, | |
| "grad_norm": 0.23478873074054718, | |
| "kl": 0.0006268819173177084, | |
| "learning_rate": 1.0109617738307911e-07, | |
| "loss": -0.0009, | |
| "num_tokens": 7179999.0, | |
| "policy_entropy_avg": 8.125, | |
| "reward": 0.23419660329818726, | |
| "reward_std": 0.5556939840316772, | |
| "rewards/cosine_scaled_reward/mean": -0.04897995665669441, | |
| "rewards/cosine_scaled_reward/std": 0.39337849617004395, | |
| "rewards/format_reward/mean": 0.3333333432674408, | |
| "rewards/format_reward/std": 0.47639307379722595, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.05714285714285714, | |
| "step": 50, | |
| "total_flos": 0.0, | |
| "train_loss": 0.00044901110231876373, | |
| "train_runtime": 4526.0548, | |
| "train_samples_per_second": 0.53, | |
| "train_steps_per_second": 0.011 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 50, | |
| "num_input_tokens_seen": 7179999, | |
| "num_train_epochs": 1, | |
| "save_steps": 50, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 4, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |