| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.5714285714285714, | |
| "eval_steps": 500, | |
| "global_step": 500, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "completion_length": 3001.9584350585938, | |
| "epoch": 0.001142857142857143, | |
| "grad_norm": 0.1892756074666977, | |
| "kl": 0.0, | |
| "learning_rate": 2e-08, | |
| "loss": -0.0, | |
| "reward": -0.05305394693277776, | |
| "reward_std": 0.17984477430582047, | |
| "rewards/cosine_scaled_reward": -0.1928562317043543, | |
| "rewards/format_reward": 0.37500000558793545, | |
| "step": 1 | |
| }, | |
| { | |
| "completion_length": 2822.541717529297, | |
| "epoch": 0.002285714285714286, | |
| "grad_norm": 0.264278769493103, | |
| "kl": 0.0, | |
| "learning_rate": 4e-08, | |
| "loss": 0.0, | |
| "reward": 0.11233407678082585, | |
| "reward_std": 0.2779897153377533, | |
| "rewards/cosine_scaled_reward": -0.009885392151772976, | |
| "rewards/format_reward": 0.4583333432674408, | |
| "step": 2 | |
| }, | |
| { | |
| "completion_length": 2963.6875610351562, | |
| "epoch": 0.0034285714285714284, | |
| "grad_norm": 0.19026866555213928, | |
| "kl": 3.454089164733887e-05, | |
| "learning_rate": 6e-08, | |
| "loss": 0.0, | |
| "reward": -0.16433760710060596, | |
| "reward_std": 0.12963848933577538, | |
| "rewards/cosine_scaled_reward": -0.30146655440330505, | |
| "rewards/format_reward": 0.27083333395421505, | |
| "step": 3 | |
| }, | |
| { | |
| "completion_length": 3083.52099609375, | |
| "epoch": 0.004571428571428572, | |
| "grad_norm": 0.18971964716911316, | |
| "kl": 2.5644898414611816e-05, | |
| "learning_rate": 8e-08, | |
| "loss": 0.0, | |
| "reward": -0.06517243073903956, | |
| "reward_std": 0.20367862656712532, | |
| "rewards/cosine_scaled_reward": -0.19678325578570366, | |
| "rewards/format_reward": 0.33333334140479565, | |
| "step": 4 | |
| }, | |
| { | |
| "completion_length": 2776.6875610351562, | |
| "epoch": 0.005714285714285714, | |
| "grad_norm": 0.19452892243862152, | |
| "kl": 3.6656856536865234e-05, | |
| "learning_rate": 1e-07, | |
| "loss": 0.0, | |
| "reward": 0.2221625093370676, | |
| "reward_std": 0.30077361315488815, | |
| "rewards/cosine_scaled_reward": 0.1190804042853415, | |
| "rewards/format_reward": 0.5000000204890966, | |
| "step": 5 | |
| }, | |
| { | |
| "completion_length": 2797.916748046875, | |
| "epoch": 0.006857142857142857, | |
| "grad_norm": 0.2365235984325409, | |
| "kl": 3.871321678161621e-05, | |
| "learning_rate": 1.2e-07, | |
| "loss": 0.0, | |
| "reward": 0.11705643311142921, | |
| "reward_std": 0.27914606034755707, | |
| "rewards/cosine_scaled_reward": -0.020885439589619637, | |
| "rewards/format_reward": 0.5208333507180214, | |
| "step": 6 | |
| }, | |
| { | |
| "completion_length": 2599.1250610351562, | |
| "epoch": 0.008, | |
| "grad_norm": 0.19415141642093658, | |
| "kl": 2.275407314300537e-05, | |
| "learning_rate": 1.4e-07, | |
| "loss": 0.0, | |
| "reward": 0.1348338108509779, | |
| "reward_std": 0.2222888581454754, | |
| "rewards/cosine_scaled_reward": 0.006776571273803711, | |
| "rewards/format_reward": 0.5, | |
| "step": 7 | |
| }, | |
| { | |
| "completion_length": 2841.3750610351562, | |
| "epoch": 0.009142857142857144, | |
| "grad_norm": 0.18534274399280548, | |
| "kl": 3.381073474884033e-05, | |
| "learning_rate": 1.6e-07, | |
| "loss": 0.0, | |
| "reward": -0.04779001139104366, | |
| "reward_std": 0.2098814733326435, | |
| "rewards/cosine_scaled_reward": -0.19928531628102064, | |
| "rewards/format_reward": 0.4166666716337204, | |
| "step": 8 | |
| }, | |
| { | |
| "completion_length": 3102.0208740234375, | |
| "epoch": 0.010285714285714285, | |
| "grad_norm": 0.20473794639110565, | |
| "kl": 4.3511390686035156e-05, | |
| "learning_rate": 1.8e-07, | |
| "loss": 0.0, | |
| "reward": 0.029760856181383133, | |
| "reward_std": 0.2512381225824356, | |
| "rewards/cosine_scaled_reward": -0.053303517401218414, | |
| "rewards/format_reward": 0.2708333432674408, | |
| "step": 9 | |
| }, | |
| { | |
| "completion_length": 2247.4376220703125, | |
| "epoch": 0.011428571428571429, | |
| "grad_norm": 0.225472092628479, | |
| "kl": 3.329664468765259e-05, | |
| "learning_rate": 2e-07, | |
| "loss": 0.0, | |
| "reward": 0.13082122802734375, | |
| "reward_std": 0.2447206862270832, | |
| "rewards/cosine_scaled_reward": -0.02737809531390667, | |
| "rewards/format_reward": 0.5833333358168602, | |
| "step": 10 | |
| }, | |
| { | |
| "completion_length": 2586.8334350585938, | |
| "epoch": 0.012571428571428572, | |
| "grad_norm": 0.2847403287887573, | |
| "kl": 3.662705421447754e-05, | |
| "learning_rate": 2.1999999999999998e-07, | |
| "loss": 0.0, | |
| "reward": 0.11637254571542144, | |
| "reward_std": 0.3530980758368969, | |
| "rewards/cosine_scaled_reward": -0.021436475217342377, | |
| "rewards/format_reward": 0.5208333432674408, | |
| "step": 11 | |
| }, | |
| { | |
| "completion_length": 2676.1251220703125, | |
| "epoch": 0.013714285714285714, | |
| "grad_norm": 0.1856570988893509, | |
| "kl": 2.3916363716125488e-05, | |
| "learning_rate": 2.4e-07, | |
| "loss": 0.0, | |
| "reward": 0.15559198334813118, | |
| "reward_std": 0.27254677191376686, | |
| "rewards/cosine_scaled_reward": 0.00797954574227333, | |
| "rewards/format_reward": 0.5833333432674408, | |
| "step": 12 | |
| }, | |
| { | |
| "completion_length": 2201.8959197998047, | |
| "epoch": 0.014857142857142857, | |
| "grad_norm": 0.21396547555923462, | |
| "kl": 2.2470951080322266e-05, | |
| "learning_rate": 2.6e-07, | |
| "loss": 0.0, | |
| "reward": 0.19927317276597023, | |
| "reward_std": 0.2963304966688156, | |
| "rewards/cosine_scaled_reward": 0.04285899642854929, | |
| "rewards/format_reward": 0.6458333432674408, | |
| "step": 13 | |
| }, | |
| { | |
| "completion_length": 2980.9584350585938, | |
| "epoch": 0.016, | |
| "grad_norm": 0.20101284980773926, | |
| "kl": 4.291534423828125e-05, | |
| "learning_rate": 2.8e-07, | |
| "loss": 0.0, | |
| "reward": 0.06336207129061222, | |
| "reward_std": 0.31606999784708023, | |
| "rewards/cosine_scaled_reward": -0.04676309134811163, | |
| "rewards/format_reward": 0.3750000074505806, | |
| "step": 14 | |
| }, | |
| { | |
| "completion_length": 3176.7500610351562, | |
| "epoch": 0.017142857142857144, | |
| "grad_norm": 0.19117361307144165, | |
| "kl": 5.060434341430664e-05, | |
| "learning_rate": 3e-07, | |
| "loss": 0.0, | |
| "reward": -0.06747908261604607, | |
| "reward_std": 0.307153781875968, | |
| "rewards/cosine_scaled_reward": -0.15347641706466675, | |
| "rewards/format_reward": 0.20833333395421505, | |
| "step": 15 | |
| }, | |
| { | |
| "completion_length": 2186.104248046875, | |
| "epoch": 0.018285714285714287, | |
| "grad_norm": 0.256155788898468, | |
| "kl": 1.3686716556549072e-05, | |
| "learning_rate": 3.2e-07, | |
| "loss": 0.0, | |
| "reward": 0.23619388276711106, | |
| "reward_std": 0.26740751788020134, | |
| "rewards/cosine_scaled_reward": 0.1003237534314394, | |
| "rewards/format_reward": 0.6041666865348816, | |
| "step": 16 | |
| }, | |
| { | |
| "completion_length": 3134.7916870117188, | |
| "epoch": 0.019428571428571427, | |
| "grad_norm": 0.17361606657505035, | |
| "kl": 4.1812658309936523e-05, | |
| "learning_rate": 3.4000000000000003e-07, | |
| "loss": 0.0, | |
| "reward": -0.10287437587976456, | |
| "reward_std": 0.14985607378184795, | |
| "rewards/cosine_scaled_reward": -0.20927279442548752, | |
| "rewards/format_reward": 0.22916667722165585, | |
| "step": 17 | |
| }, | |
| { | |
| "completion_length": 3086.3959350585938, | |
| "epoch": 0.02057142857142857, | |
| "grad_norm": 0.23153060674667358, | |
| "kl": 5.2809715270996094e-05, | |
| "learning_rate": 3.6e-07, | |
| "loss": 0.0, | |
| "reward": -0.048283468931913376, | |
| "reward_std": 0.28065528720617294, | |
| "rewards/cosine_scaled_reward": -0.15300771407783031, | |
| "rewards/format_reward": 0.27083333395421505, | |
| "step": 18 | |
| }, | |
| { | |
| "completion_length": 3197.229248046875, | |
| "epoch": 0.021714285714285714, | |
| "grad_norm": 0.15150664746761322, | |
| "kl": 2.8431415557861328e-05, | |
| "learning_rate": 3.7999999999999996e-07, | |
| "loss": 0.0, | |
| "reward": -0.04134747572243214, | |
| "reward_std": 0.25655536726117134, | |
| "rewards/cosine_scaled_reward": -0.1307651773095131, | |
| "rewards/format_reward": 0.2291666679084301, | |
| "step": 19 | |
| }, | |
| { | |
| "completion_length": 2551.729217529297, | |
| "epoch": 0.022857142857142857, | |
| "grad_norm": 0.23382136225700378, | |
| "kl": 3.3795833587646484e-05, | |
| "learning_rate": 4e-07, | |
| "loss": 0.0, | |
| "reward": 0.06840577768161893, | |
| "reward_std": 0.2175830602645874, | |
| "rewards/cosine_scaled_reward": -0.08142943307757378, | |
| "rewards/format_reward": 0.5000000149011612, | |
| "step": 20 | |
| }, | |
| { | |
| "completion_length": 2715.0626220703125, | |
| "epoch": 0.024, | |
| "grad_norm": 0.2693728506565094, | |
| "kl": 2.7894973754882812e-05, | |
| "learning_rate": 4.1999999999999995e-07, | |
| "loss": 0.0, | |
| "reward": 0.09808723395690322, | |
| "reward_std": 0.2364928014576435, | |
| "rewards/cosine_scaled_reward": -0.02434827946126461, | |
| "rewards/format_reward": 0.45833333395421505, | |
| "step": 21 | |
| }, | |
| { | |
| "completion_length": 3276.0625610351562, | |
| "epoch": 0.025142857142857144, | |
| "grad_norm": 0.19304966926574707, | |
| "kl": 3.9517879486083984e-05, | |
| "learning_rate": 4.3999999999999997e-07, | |
| "loss": 0.0, | |
| "reward": -0.08807891490869224, | |
| "reward_std": 0.2701016888022423, | |
| "rewards/cosine_scaled_reward": -0.1670425608754158, | |
| "rewards/format_reward": 0.1666666716337204, | |
| "step": 22 | |
| }, | |
| { | |
| "completion_length": 3304.3959350585938, | |
| "epoch": 0.026285714285714287, | |
| "grad_norm": 0.15851277112960815, | |
| "kl": 5.078315734863281e-05, | |
| "learning_rate": 4.6e-07, | |
| "loss": 0.0, | |
| "reward": -0.04373410594416782, | |
| "reward_std": 0.24115629866719246, | |
| "rewards/cosine_scaled_reward": -0.1398780420422554, | |
| "rewards/format_reward": 0.2500000111758709, | |
| "step": 23 | |
| }, | |
| { | |
| "completion_length": 2280.3333740234375, | |
| "epoch": 0.027428571428571427, | |
| "grad_norm": 0.21461711823940277, | |
| "kl": 8.79727303981781e-06, | |
| "learning_rate": 4.8e-07, | |
| "loss": 0.0, | |
| "reward": 0.2311121616512537, | |
| "reward_std": 0.23989838548004627, | |
| "rewards/cosine_scaled_reward": 0.09470203705132008, | |
| "rewards/format_reward": 0.6041666772216558, | |
| "step": 24 | |
| }, | |
| { | |
| "completion_length": 2743.0416870117188, | |
| "epoch": 0.02857142857142857, | |
| "grad_norm": 0.2246316522359848, | |
| "kl": 2.2739171981811523e-05, | |
| "learning_rate": 5e-07, | |
| "loss": 0.0, | |
| "reward": 0.10195200517773628, | |
| "reward_std": 0.2438894398510456, | |
| "rewards/cosine_scaled_reward": 0.006860721856355667, | |
| "rewards/format_reward": 0.3750000037252903, | |
| "step": 25 | |
| }, | |
| { | |
| "completion_length": 3110.2500610351562, | |
| "epoch": 0.029714285714285714, | |
| "grad_norm": 0.17850153148174286, | |
| "kl": 2.816319465637207e-05, | |
| "learning_rate": 5.2e-07, | |
| "loss": 0.0, | |
| "reward": 0.12602470815181732, | |
| "reward_std": 0.3109820708632469, | |
| "rewards/cosine_scaled_reward": 0.0360703282058239, | |
| "rewards/format_reward": 0.3750000149011612, | |
| "step": 26 | |
| }, | |
| { | |
| "completion_length": 3011.854248046875, | |
| "epoch": 0.030857142857142857, | |
| "grad_norm": 0.1936245560646057, | |
| "kl": 3.886222839355469e-05, | |
| "learning_rate": 5.4e-07, | |
| "loss": 0.0, | |
| "reward": 0.0031234845519065857, | |
| "reward_std": 0.15648237243294716, | |
| "rewards/cosine_scaled_reward": -0.07742472551763058, | |
| "rewards/format_reward": 0.25, | |
| "step": 27 | |
| }, | |
| { | |
| "completion_length": 3133.7708740234375, | |
| "epoch": 0.032, | |
| "grad_norm": 0.19556812942028046, | |
| "kl": 4.5865774154663086e-05, | |
| "learning_rate": 5.6e-07, | |
| "loss": 0.0, | |
| "reward": 0.05877143098041415, | |
| "reward_std": 0.32328512519598007, | |
| "rewards/cosine_scaled_reward": -0.043721283320337534, | |
| "rewards/format_reward": 0.35416667722165585, | |
| "step": 28 | |
| }, | |
| { | |
| "completion_length": 3163.6666870117188, | |
| "epoch": 0.03314285714285714, | |
| "grad_norm": 0.16339358687400818, | |
| "kl": 2.5033950805664062e-05, | |
| "learning_rate": 5.8e-07, | |
| "loss": 0.0, | |
| "reward": 0.027387002483010292, | |
| "reward_std": 0.337462954223156, | |
| "rewards/cosine_scaled_reward": -0.07489126734435558, | |
| "rewards/format_reward": 0.3333333469927311, | |
| "step": 29 | |
| }, | |
| { | |
| "completion_length": 3104.3959350585938, | |
| "epoch": 0.03428571428571429, | |
| "grad_norm": 0.20371638238430023, | |
| "kl": 3.0308961868286133e-05, | |
| "learning_rate": 6e-07, | |
| "loss": 0.0, | |
| "reward": 0.08465440198779106, | |
| "reward_std": 0.2969431169331074, | |
| "rewards/cosine_scaled_reward": -0.006935901008546352, | |
| "rewards/format_reward": 0.3541666828095913, | |
| "step": 30 | |
| }, | |
| { | |
| "completion_length": 2491.2708740234375, | |
| "epoch": 0.03542857142857143, | |
| "grad_norm": 0.3123537600040436, | |
| "kl": 4.607439041137695e-05, | |
| "learning_rate": 6.2e-07, | |
| "loss": 0.0, | |
| "reward": -0.005427001044154167, | |
| "reward_std": 0.19760142639279366, | |
| "rewards/cosine_scaled_reward": -0.1529179997742176, | |
| "rewards/format_reward": 0.4375000074505806, | |
| "step": 31 | |
| }, | |
| { | |
| "completion_length": 3482.5208740234375, | |
| "epoch": 0.036571428571428574, | |
| "grad_norm": 0.15471217036247253, | |
| "kl": 3.0040740966796875e-05, | |
| "learning_rate": 6.4e-07, | |
| "loss": 0.0, | |
| "reward": -0.12795986607670784, | |
| "reward_std": 0.22705752402544022, | |
| "rewards/cosine_scaled_reward": -0.20094125345349312, | |
| "rewards/format_reward": 0.1041666679084301, | |
| "step": 32 | |
| }, | |
| { | |
| "completion_length": 2918.4166870117188, | |
| "epoch": 0.037714285714285714, | |
| "grad_norm": 0.1735975742340088, | |
| "kl": 2.60770320892334e-05, | |
| "learning_rate": 6.6e-07, | |
| "loss": 0.0, | |
| "reward": 0.13466424029320478, | |
| "reward_std": 0.30455923825502396, | |
| "rewards/cosine_scaled_reward": 0.016905852127820253, | |
| "rewards/format_reward": 0.4791666828095913, | |
| "step": 33 | |
| }, | |
| { | |
| "completion_length": 3103.1875610351562, | |
| "epoch": 0.038857142857142854, | |
| "grad_norm": 0.20565363764762878, | |
| "kl": 2.7477741241455078e-05, | |
| "learning_rate": 6.800000000000001e-07, | |
| "loss": 0.0, | |
| "reward": -0.0920376293361187, | |
| "reward_std": 0.22466492280364037, | |
| "rewards/cosine_scaled_reward": -0.19513528188690543, | |
| "rewards/format_reward": 0.2291666716337204, | |
| "step": 34 | |
| }, | |
| { | |
| "completion_length": 2871.791748046875, | |
| "epoch": 0.04, | |
| "grad_norm": 0.16891056299209595, | |
| "kl": 3.1381845474243164e-05, | |
| "learning_rate": 7e-07, | |
| "loss": 0.0, | |
| "reward": 0.09445883147418499, | |
| "reward_std": 0.32046519964933395, | |
| "rewards/cosine_scaled_reward": -0.028155114501714706, | |
| "rewards/format_reward": 0.43750001303851604, | |
| "step": 35 | |
| }, | |
| { | |
| "completion_length": 2533.5625915527344, | |
| "epoch": 0.04114285714285714, | |
| "grad_norm": 0.19785140454769135, | |
| "kl": 2.5272369384765625e-05, | |
| "learning_rate": 7.2e-07, | |
| "loss": 0.0, | |
| "reward": 0.19070486724376678, | |
| "reward_std": 0.13970058038830757, | |
| "rewards/cosine_scaled_reward": 0.08542170003056526, | |
| "rewards/format_reward": 0.47916667722165585, | |
| "step": 36 | |
| }, | |
| { | |
| "completion_length": 2755.2083740234375, | |
| "epoch": 0.04228571428571429, | |
| "grad_norm": 0.21290314197540283, | |
| "kl": 2.9027462005615234e-05, | |
| "learning_rate": 7.4e-07, | |
| "loss": 0.0, | |
| "reward": 0.02860638126730919, | |
| "reward_std": 0.2937544248998165, | |
| "rewards/cosine_scaled_reward": -0.1261284868232906, | |
| "rewards/format_reward": 0.4791666865348816, | |
| "step": 37 | |
| }, | |
| { | |
| "completion_length": 3023.4376220703125, | |
| "epoch": 0.04342857142857143, | |
| "grad_norm": 0.19873568415641785, | |
| "kl": 3.618001937866211e-05, | |
| "learning_rate": 7.599999999999999e-07, | |
| "loss": 0.0, | |
| "reward": -0.013171980390325189, | |
| "reward_std": 0.20296454429626465, | |
| "rewards/cosine_scaled_reward": -0.1557461880147457, | |
| "rewards/format_reward": 0.416666679084301, | |
| "step": 38 | |
| }, | |
| { | |
| "completion_length": 2770.166748046875, | |
| "epoch": 0.044571428571428574, | |
| "grad_norm": 0.21600858867168427, | |
| "kl": 3.2961368560791016e-05, | |
| "learning_rate": 7.799999999999999e-07, | |
| "loss": 0.0, | |
| "reward": 0.04621700069401413, | |
| "reward_std": 0.3573795333504677, | |
| "rewards/cosine_scaled_reward": -0.0878668837249279, | |
| "rewards/format_reward": 0.4375, | |
| "step": 39 | |
| }, | |
| { | |
| "completion_length": 2921.8334350585938, | |
| "epoch": 0.045714285714285714, | |
| "grad_norm": 0.1665465086698532, | |
| "kl": 1.576542854309082e-05, | |
| "learning_rate": 8e-07, | |
| "loss": 0.0, | |
| "reward": -0.08147826488129795, | |
| "reward_std": 0.2727391682565212, | |
| "rewards/cosine_scaled_reward": -0.23862353712320328, | |
| "rewards/format_reward": 0.3958333432674408, | |
| "step": 40 | |
| }, | |
| { | |
| "completion_length": 3219.4375, | |
| "epoch": 0.046857142857142854, | |
| "grad_norm": 0.15641310811042786, | |
| "kl": 2.47955322265625e-05, | |
| "learning_rate": 8.199999999999999e-07, | |
| "loss": 0.0, | |
| "reward": 0.04313786077545956, | |
| "reward_std": 0.2788317874073982, | |
| "rewards/cosine_scaled_reward": -0.045081598684191704, | |
| "rewards/format_reward": 0.2916666716337204, | |
| "step": 41 | |
| }, | |
| { | |
| "completion_length": 2030.6250610351562, | |
| "epoch": 0.048, | |
| "grad_norm": 0.3318580389022827, | |
| "kl": 5.9932470321655273e-05, | |
| "learning_rate": 8.399999999999999e-07, | |
| "loss": 0.0, | |
| "reward": 0.1814209260046482, | |
| "reward_std": 0.24633333832025528, | |
| "rewards/cosine_scaled_reward": -0.004054427146911621, | |
| "rewards/format_reward": 0.7083333507180214, | |
| "step": 42 | |
| }, | |
| { | |
| "completion_length": 2986.7084350585938, | |
| "epoch": 0.04914285714285714, | |
| "grad_norm": 0.18334151804447174, | |
| "kl": 2.3711472749710083e-05, | |
| "learning_rate": 8.599999999999999e-07, | |
| "loss": 0.0, | |
| "reward": 0.022090405225753784, | |
| "reward_std": 0.3203391507267952, | |
| "rewards/cosine_scaled_reward": -0.10132655501365662, | |
| "rewards/format_reward": 0.39583334140479565, | |
| "step": 43 | |
| }, | |
| { | |
| "completion_length": 2783.3959350585938, | |
| "epoch": 0.05028571428571429, | |
| "grad_norm": 0.2206653207540512, | |
| "kl": 0.00010889768600463867, | |
| "learning_rate": 8.799999999999999e-07, | |
| "loss": 0.0, | |
| "reward": 0.13657060265541077, | |
| "reward_std": 0.29330166801810265, | |
| "rewards/cosine_scaled_reward": 0.028908416628837585, | |
| "rewards/format_reward": 0.4375000260770321, | |
| "step": 44 | |
| }, | |
| { | |
| "completion_length": 2881.1250610351562, | |
| "epoch": 0.05142857142857143, | |
| "grad_norm": 0.17664436995983124, | |
| "kl": 2.9489398002624512e-05, | |
| "learning_rate": 9e-07, | |
| "loss": 0.0, | |
| "reward": 0.2449277015402913, | |
| "reward_std": 0.40640798956155777, | |
| "rewards/cosine_scaled_reward": 0.17139457043958828, | |
| "rewards/format_reward": 0.4375000111758709, | |
| "step": 45 | |
| }, | |
| { | |
| "completion_length": 3004.812530517578, | |
| "epoch": 0.052571428571428575, | |
| "grad_norm": 0.17481741309165955, | |
| "kl": 1.0464340448379517e-05, | |
| "learning_rate": 9.2e-07, | |
| "loss": 0.0, | |
| "reward": 0.054059505462646484, | |
| "reward_std": 0.1310178069397807, | |
| "rewards/cosine_scaled_reward": -0.04911454766988754, | |
| "rewards/format_reward": 0.3541666716337204, | |
| "step": 46 | |
| }, | |
| { | |
| "completion_length": 2466.6458740234375, | |
| "epoch": 0.053714285714285714, | |
| "grad_norm": 0.2526704668998718, | |
| "kl": 2.4911249056458473e-05, | |
| "learning_rate": 9.399999999999999e-07, | |
| "loss": 0.0, | |
| "reward": 0.14142334461212158, | |
| "reward_std": 0.30394549667835236, | |
| "rewards/cosine_scaled_reward": -0.018136408179998398, | |
| "rewards/format_reward": 0.6041666865348816, | |
| "step": 47 | |
| }, | |
| { | |
| "completion_length": 2828.0833740234375, | |
| "epoch": 0.054857142857142854, | |
| "grad_norm": 0.1843711882829666, | |
| "kl": 5.0902366638183594e-05, | |
| "learning_rate": 9.6e-07, | |
| "loss": 0.0, | |
| "reward": 0.17658406496047974, | |
| "reward_std": 0.41882630810141563, | |
| "rewards/cosine_scaled_reward": 0.07139459624886513, | |
| "rewards/format_reward": 0.4583333432674408, | |
| "step": 48 | |
| }, | |
| { | |
| "completion_length": 2165.8541870117188, | |
| "epoch": 0.056, | |
| "grad_norm": 0.2018611580133438, | |
| "kl": 8.557736873626709e-05, | |
| "learning_rate": 9.8e-07, | |
| "loss": 0.0, | |
| "reward": 0.2922453209757805, | |
| "reward_std": 0.29717790707945824, | |
| "rewards/cosine_scaled_reward": 0.1657434180378914, | |
| "rewards/format_reward": 0.625, | |
| "step": 49 | |
| }, | |
| { | |
| "completion_length": 2188.0834350585938, | |
| "epoch": 0.05714285714285714, | |
| "grad_norm": 0.34193405508995056, | |
| "kl": 0.00023797806352376938, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0, | |
| "reward": 0.2388845265377313, | |
| "reward_std": 0.29164354503154755, | |
| "rewards/cosine_scaled_reward": 0.10962018743157387, | |
| "rewards/format_reward": 0.583333358168602, | |
| "step": 50 | |
| }, | |
| { | |
| "completion_length": 3217.541748046875, | |
| "epoch": 0.05828571428571429, | |
| "grad_norm": 0.16055454313755035, | |
| "kl": 5.5164098739624023e-05, | |
| "learning_rate": 9.999890338174275e-07, | |
| "loss": 0.0, | |
| "reward": -0.053118061856366694, | |
| "reward_std": 0.2539052776992321, | |
| "rewards/cosine_scaled_reward": -0.1391819417476654, | |
| "rewards/format_reward": 0.20833334513008595, | |
| "step": 51 | |
| }, | |
| { | |
| "completion_length": 3114.9584350585938, | |
| "epoch": 0.05942857142857143, | |
| "grad_norm": 0.16836906969547272, | |
| "kl": 0.00012135505676269531, | |
| "learning_rate": 9.999561358041868e-07, | |
| "loss": 0.0, | |
| "reward": -0.03695523925125599, | |
| "reward_std": 0.2873278111219406, | |
| "rewards/cosine_scaled_reward": -0.15903650410473347, | |
| "rewards/format_reward": 0.3333333507180214, | |
| "step": 52 | |
| }, | |
| { | |
| "completion_length": 2896.8541870117188, | |
| "epoch": 0.060571428571428575, | |
| "grad_norm": 0.2661495506763458, | |
| "kl": 0.00020489096641540527, | |
| "learning_rate": 9.999013075636804e-07, | |
| "loss": 0.0, | |
| "reward": -0.06146936025470495, | |
| "reward_std": 0.17227579653263092, | |
| "rewards/cosine_scaled_reward": -0.18547281250357628, | |
| "rewards/format_reward": 0.3125, | |
| "step": 53 | |
| }, | |
| { | |
| "completion_length": 2592.7708740234375, | |
| "epoch": 0.061714285714285715, | |
| "grad_norm": 0.18709887564182281, | |
| "kl": 7.784366607666016e-05, | |
| "learning_rate": 9.998245517681593e-07, | |
| "loss": 0.0, | |
| "reward": -0.08744588773697615, | |
| "reward_std": 0.13443115912377834, | |
| "rewards/cosine_scaled_reward": -0.2370000034570694, | |
| "rewards/format_reward": 0.375, | |
| "step": 54 | |
| }, | |
| { | |
| "completion_length": 3108.9583740234375, | |
| "epoch": 0.06285714285714286, | |
| "grad_norm": 0.17392441630363464, | |
| "kl": 0.00017219781875610352, | |
| "learning_rate": 9.997258721585931e-07, | |
| "loss": 0.0, | |
| "reward": -0.04910636320710182, | |
| "reward_std": 0.23171953111886978, | |
| "rewards/cosine_scaled_reward": -0.15009244764223695, | |
| "rewards/format_reward": 0.25000000558793545, | |
| "step": 55 | |
| }, | |
| { | |
| "completion_length": 3085.729248046875, | |
| "epoch": 0.064, | |
| "grad_norm": 0.1863940954208374, | |
| "kl": 0.00041985511779785156, | |
| "learning_rate": 9.996052735444862e-07, | |
| "loss": 0.0, | |
| "reward": 0.0016605854034423828, | |
| "reward_std": 0.26457204297184944, | |
| "rewards/cosine_scaled_reward": -0.09207657910883427, | |
| "rewards/format_reward": 0.2916666716337204, | |
| "step": 56 | |
| }, | |
| { | |
| "completion_length": 2939.916717529297, | |
| "epoch": 0.06514285714285714, | |
| "grad_norm": 0.21433454751968384, | |
| "kl": 0.0003230571746826172, | |
| "learning_rate": 9.994627618036452e-07, | |
| "loss": 0.0, | |
| "reward": -0.0446708258241415, | |
| "reward_std": 0.2267843373119831, | |
| "rewards/cosine_scaled_reward": -0.16660560760647058, | |
| "rewards/format_reward": 0.3333333395421505, | |
| "step": 57 | |
| }, | |
| { | |
| "completion_length": 3045.0626220703125, | |
| "epoch": 0.06628571428571428, | |
| "grad_norm": 0.2040744423866272, | |
| "kl": 0.0006006956100463867, | |
| "learning_rate": 9.992983438818915e-07, | |
| "loss": 0.0, | |
| "reward": -0.0063889771699905396, | |
| "reward_std": 0.32527972012758255, | |
| "rewards/cosine_scaled_reward": -0.1043020358774811, | |
| "rewards/format_reward": 0.2916666716337204, | |
| "step": 58 | |
| }, | |
| { | |
| "completion_length": 2990.0000610351562, | |
| "epoch": 0.06742857142857143, | |
| "grad_norm": 0.21800561249256134, | |
| "kl": 0.0008556842803955078, | |
| "learning_rate": 9.991120277927223e-07, | |
| "loss": 0.0, | |
| "reward": 0.11553543619811535, | |
| "reward_std": 0.2411040961742401, | |
| "rewards/cosine_scaled_reward": 0.03165358677506447, | |
| "rewards/format_reward": 0.35416667722165585, | |
| "step": 59 | |
| }, | |
| { | |
| "completion_length": 2764.4376220703125, | |
| "epoch": 0.06857142857142857, | |
| "grad_norm": 0.19482655823230743, | |
| "kl": 0.0002484321594238281, | |
| "learning_rate": 9.989038226169207e-07, | |
| "loss": 0.0, | |
| "reward": 0.11085247556911781, | |
| "reward_std": 0.19889794662594795, | |
| "rewards/cosine_scaled_reward": 0.03661669138818979, | |
| "rewards/format_reward": 0.31250000186264515, | |
| "step": 60 | |
| }, | |
| { | |
| "completion_length": 2435.7084045410156, | |
| "epoch": 0.06971428571428571, | |
| "grad_norm": 0.19535353779792786, | |
| "kl": 0.0001939535140991211, | |
| "learning_rate": 9.98673738502114e-07, | |
| "loss": 0.0, | |
| "reward": 0.20854798704385757, | |
| "reward_std": 0.2243325486779213, | |
| "rewards/cosine_scaled_reward": 0.0953704472631216, | |
| "rewards/format_reward": 0.5208333432674408, | |
| "step": 61 | |
| }, | |
| { | |
| "completion_length": 3321.3959350585938, | |
| "epoch": 0.07085714285714285, | |
| "grad_norm": 0.17011719942092896, | |
| "kl": 0.0002614259719848633, | |
| "learning_rate": 9.98421786662277e-07, | |
| "loss": 0.0, | |
| "reward": 0.07449256628751755, | |
| "reward_std": 0.320609487593174, | |
| "rewards/cosine_scaled_reward": -0.013772495090961456, | |
| "rewards/format_reward": 0.3333333432674408, | |
| "step": 62 | |
| }, | |
| { | |
| "completion_length": 2038.0000610351562, | |
| "epoch": 0.072, | |
| "grad_norm": 0.22595028579235077, | |
| "kl": 0.002492666244506836, | |
| "learning_rate": 9.981479793771866e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1877956110984087, | |
| "reward_std": 0.2516198009252548, | |
| "rewards/cosine_scaled_reward": 0.009264111518859863, | |
| "rewards/format_reward": 0.6875000074505806, | |
| "step": 63 | |
| }, | |
| { | |
| "completion_length": 3070.6666870117188, | |
| "epoch": 0.07314285714285715, | |
| "grad_norm": 0.15192179381847382, | |
| "kl": 0.00020563602447509766, | |
| "learning_rate": 9.97852329991824e-07, | |
| "loss": 0.0, | |
| "reward": 0.14650224149227142, | |
| "reward_std": 0.3490786999464035, | |
| "rewards/cosine_scaled_reward": 0.059755959548056126, | |
| "rewards/format_reward": 0.39583334885537624, | |
| "step": 64 | |
| }, | |
| { | |
| "completion_length": 2546.2709350585938, | |
| "epoch": 0.07428571428571429, | |
| "grad_norm": 0.18594862520694733, | |
| "kl": 0.0012459754943847656, | |
| "learning_rate": 9.975348529157229e-07, | |
| "loss": 0.0, | |
| "reward": 0.11259761109249666, | |
| "reward_std": 0.30045806616544724, | |
| "rewards/cosine_scaled_reward": -0.027794551104307175, | |
| "rewards/format_reward": 0.5208333432674408, | |
| "step": 65 | |
| }, | |
| { | |
| "completion_length": 2988.0, | |
| "epoch": 0.07542857142857143, | |
| "grad_norm": 0.16415195167064667, | |
| "kl": 0.0010949969291687012, | |
| "learning_rate": 9.971955636222684e-07, | |
| "loss": 0.0, | |
| "reward": -0.034010959789156914, | |
| "reward_std": 0.27774763107299805, | |
| "rewards/cosine_scaled_reward": -0.16830014809966087, | |
| "rewards/format_reward": 0.37500000186264515, | |
| "step": 66 | |
| }, | |
| { | |
| "completion_length": 3065.8333740234375, | |
| "epoch": 0.07657142857142857, | |
| "grad_norm": 0.1508760303258896, | |
| "kl": 0.0005504488945007324, | |
| "learning_rate": 9.968344786479415e-07, | |
| "loss": 0.0, | |
| "reward": 0.17344105429947376, | |
| "reward_std": 0.3407328389585018, | |
| "rewards/cosine_scaled_reward": 0.0939273051917553, | |
| "rewards/format_reward": 0.39583334885537624, | |
| "step": 67 | |
| }, | |
| { | |
| "completion_length": 2521.9583740234375, | |
| "epoch": 0.07771428571428571, | |
| "grad_norm": 0.2780892252922058, | |
| "kl": 0.0031766891479492188, | |
| "learning_rate": 9.964516155915151e-07, | |
| "loss": 0.0001, | |
| "reward": -0.006004307419061661, | |
| "reward_std": 0.21513504162430763, | |
| "rewards/cosine_scaled_reward": -0.142730213701725, | |
| "rewards/format_reward": 0.3958333432674408, | |
| "step": 68 | |
| }, | |
| { | |
| "completion_length": 2483.6875610351562, | |
| "epoch": 0.07885714285714286, | |
| "grad_norm": 0.1821538656949997, | |
| "kl": 0.000514984130859375, | |
| "learning_rate": 9.960469931131936e-07, | |
| "loss": 0.0, | |
| "reward": 0.13737047230824828, | |
| "reward_std": 0.2913679927587509, | |
| "rewards/cosine_scaled_reward": 0.00387752428650856, | |
| "rewards/format_reward": 0.5208333432674408, | |
| "step": 69 | |
| }, | |
| { | |
| "completion_length": 3145.9791870117188, | |
| "epoch": 0.08, | |
| "grad_norm": 0.1587502807378769, | |
| "kl": 0.0017343759536743164, | |
| "learning_rate": 9.956206309337066e-07, | |
| "loss": 0.0001, | |
| "reward": 0.10869743674993515, | |
| "reward_std": 0.26583526469767094, | |
| "rewards/cosine_scaled_reward": 0.0119447261095047, | |
| "rewards/format_reward": 0.3958333432674408, | |
| "step": 70 | |
| }, | |
| { | |
| "completion_length": 2392.4584045410156, | |
| "epoch": 0.08114285714285714, | |
| "grad_norm": 0.2353833019733429, | |
| "kl": 0.0014793872833251953, | |
| "learning_rate": 9.951725498333448e-07, | |
| "loss": 0.0001, | |
| "reward": 0.06978913443163037, | |
| "reward_std": 0.23073862865567207, | |
| "rewards/cosine_scaled_reward": -0.08474473655223846, | |
| "rewards/format_reward": 0.5208333432674408, | |
| "step": 71 | |
| }, | |
| { | |
| "completion_length": 2445.25, | |
| "epoch": 0.08228571428571428, | |
| "grad_norm": 0.36222851276397705, | |
| "kl": 0.007985234260559082, | |
| "learning_rate": 9.947027716509488e-07, | |
| "loss": 0.0003, | |
| "reward": 0.28675128147006035, | |
| "reward_std": 0.26310405880212784, | |
| "rewards/cosine_scaled_reward": 0.15304938424378633, | |
| "rewards/format_reward": 0.645833358168602, | |
| "step": 72 | |
| }, | |
| { | |
| "completion_length": 2111.916717529297, | |
| "epoch": 0.08342857142857144, | |
| "grad_norm": 0.2483416646718979, | |
| "kl": 0.0014190673828125, | |
| "learning_rate": 9.942113192828444e-07, | |
| "loss": 0.0001, | |
| "reward": 0.2418079525232315, | |
| "reward_std": 0.23962385952472687, | |
| "rewards/cosine_scaled_reward": 0.0939856581389904, | |
| "rewards/format_reward": 0.6458333432674408, | |
| "step": 73 | |
| }, | |
| { | |
| "completion_length": 2321.0625915527344, | |
| "epoch": 0.08457142857142858, | |
| "grad_norm": 0.20480823516845703, | |
| "kl": 0.0015668869018554688, | |
| "learning_rate": 9.93698216681727e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1605784334242344, | |
| "reward_std": 0.3177975155413151, | |
| "rewards/cosine_scaled_reward": 0.00783238559961319, | |
| "rewards/format_reward": 0.6041666865348816, | |
| "step": 74 | |
| }, | |
| { | |
| "completion_length": 2604.187530517578, | |
| "epoch": 0.08571428571428572, | |
| "grad_norm": 0.2458542287349701, | |
| "kl": 0.0021820068359375, | |
| "learning_rate": 9.931634888554935e-07, | |
| "loss": 0.0001, | |
| "reward": 0.02317179925739765, | |
| "reward_std": 0.24085771292448044, | |
| "rewards/cosine_scaled_reward": -0.10294242203235626, | |
| "rewards/format_reward": 0.3958333395421505, | |
| "step": 75 | |
| }, | |
| { | |
| "completion_length": 2759.1459350585938, | |
| "epoch": 0.08685714285714285, | |
| "grad_norm": 0.19815924763679504, | |
| "kl": 0.0006747245788574219, | |
| "learning_rate": 9.926071618660237e-07, | |
| "loss": 0.0, | |
| "reward": 0.17977549135684967, | |
| "reward_std": 0.22920313104987144, | |
| "rewards/cosine_scaled_reward": 0.060693852603435516, | |
| "rewards/format_reward": 0.5000000074505806, | |
| "step": 76 | |
| }, | |
| { | |
| "completion_length": 3166.2500610351562, | |
| "epoch": 0.088, | |
| "grad_norm": 0.17464476823806763, | |
| "kl": 0.000640869140625, | |
| "learning_rate": 9.9202926282791e-07, | |
| "loss": 0.0, | |
| "reward": 0.0053346361964941025, | |
| "reward_std": 0.2872692942619324, | |
| "rewards/cosine_scaled_reward": -0.07941475044935942, | |
| "rewards/format_reward": 0.2500000037252903, | |
| "step": 77 | |
| }, | |
| { | |
| "completion_length": 2879.6875610351562, | |
| "epoch": 0.08914285714285715, | |
| "grad_norm": 0.17484599351882935, | |
| "kl": 0.0008184909820556641, | |
| "learning_rate": 9.91429819907136e-07, | |
| "loss": 0.0, | |
| "reward": 0.031344905495643616, | |
| "reward_std": 0.2594750449061394, | |
| "rewards/cosine_scaled_reward": -0.07576533406972885, | |
| "rewards/format_reward": 0.35416667722165585, | |
| "step": 78 | |
| }, | |
| { | |
| "completion_length": 2861.7083740234375, | |
| "epoch": 0.09028571428571429, | |
| "grad_norm": 0.18350628018379211, | |
| "kl": 0.0004169940948486328, | |
| "learning_rate": 9.908088623197048e-07, | |
| "loss": 0.0, | |
| "reward": 0.13247722014784813, | |
| "reward_std": 0.2174980491399765, | |
| "rewards/cosine_scaled_reward": 0.024736488237977028, | |
| "rewards/format_reward": 0.4375000074505806, | |
| "step": 79 | |
| }, | |
| { | |
| "completion_length": 3313.1041870117188, | |
| "epoch": 0.09142857142857143, | |
| "grad_norm": 0.15940068662166595, | |
| "kl": 0.00030362606048583984, | |
| "learning_rate": 9.901664203302124e-07, | |
| "loss": 0.0, | |
| "reward": -0.07443897030316293, | |
| "reward_std": 0.20227612555027008, | |
| "rewards/cosine_scaled_reward": -0.16378207132220268, | |
| "rewards/format_reward": 0.2083333395421505, | |
| "step": 80 | |
| }, | |
| { | |
| "completion_length": 3127.8750610351562, | |
| "epoch": 0.09257142857142857, | |
| "grad_norm": 0.17740540206432343, | |
| "kl": 0.0021224021911621094, | |
| "learning_rate": 9.895025252503755e-07, | |
| "loss": 0.0001, | |
| "reward": 0.025630776770412922, | |
| "reward_std": 0.31113822758197784, | |
| "rewards/cosine_scaled_reward": -0.04427819326519966, | |
| "rewards/format_reward": 0.22916666977107525, | |
| "step": 81 | |
| }, | |
| { | |
| "completion_length": 2915.4791870117188, | |
| "epoch": 0.09371428571428571, | |
| "grad_norm": 0.17501351237297058, | |
| "kl": 0.0007612705230712891, | |
| "learning_rate": 9.888172094375033e-07, | |
| "loss": 0.0, | |
| "reward": -0.0019456367008388042, | |
| "reward_std": 0.26062510162591934, | |
| "rewards/cosine_scaled_reward": -0.13531204964965582, | |
| "rewards/format_reward": 0.3958333432674408, | |
| "step": 82 | |
| }, | |
| { | |
| "completion_length": 3508.1875610351562, | |
| "epoch": 0.09485714285714286, | |
| "grad_norm": 0.15616776049137115, | |
| "kl": 0.0002772808074951172, | |
| "learning_rate": 9.881105062929221e-07, | |
| "loss": 0.0, | |
| "reward": -0.13871120987460017, | |
| "reward_std": 0.21259715221822262, | |
| "rewards/cosine_scaled_reward": -0.20505784079432487, | |
| "rewards/format_reward": 0.0833333358168602, | |
| "step": 83 | |
| }, | |
| { | |
| "completion_length": 2658.7708740234375, | |
| "epoch": 0.096, | |
| "grad_norm": 0.19438283145427704, | |
| "kl": 0.000476837158203125, | |
| "learning_rate": 9.873824502603459e-07, | |
| "loss": 0.0, | |
| "reward": 0.2181803850689903, | |
| "reward_std": 0.32128985971212387, | |
| "rewards/cosine_scaled_reward": 0.09941665828227997, | |
| "rewards/format_reward": 0.5416666716337204, | |
| "step": 84 | |
| }, | |
| { | |
| "completion_length": 3100.2083740234375, | |
| "epoch": 0.09714285714285714, | |
| "grad_norm": 0.17281532287597656, | |
| "kl": 0.0006203651428222656, | |
| "learning_rate": 9.866330768241983e-07, | |
| "loss": 0.0, | |
| "reward": 0.0023652464151382446, | |
| "reward_std": 0.27234018221497536, | |
| "rewards/cosine_scaled_reward": -0.0952380420640111, | |
| "rewards/format_reward": 0.2916666753590107, | |
| "step": 85 | |
| }, | |
| { | |
| "completion_length": 2836.354248046875, | |
| "epoch": 0.09828571428571428, | |
| "grad_norm": 0.16363169252872467, | |
| "kl": 0.0010099411010742188, | |
| "learning_rate": 9.85862422507884e-07, | |
| "loss": 0.0, | |
| "reward": -0.03766683675348759, | |
| "reward_std": 0.21276579797267914, | |
| "rewards/cosine_scaled_reward": -0.1654567075893283, | |
| "rewards/format_reward": 0.3541666716337204, | |
| "step": 86 | |
| }, | |
| { | |
| "completion_length": 3115.354248046875, | |
| "epoch": 0.09942857142857142, | |
| "grad_norm": 0.18644234538078308, | |
| "kl": 0.002384185791015625, | |
| "learning_rate": 9.850705248720068e-07, | |
| "loss": 0.0001, | |
| "reward": -0.032251302152872086, | |
| "reward_std": 0.24311810731887817, | |
| "rewards/cosine_scaled_reward": -0.1445614593103528, | |
| "rewards/format_reward": 0.3125000074505806, | |
| "step": 87 | |
| }, | |
| { | |
| "completion_length": 2530.5833740234375, | |
| "epoch": 0.10057142857142858, | |
| "grad_norm": 0.19104546308517456, | |
| "kl": 0.0012140274047851562, | |
| "learning_rate": 9.8425742251254e-07, | |
| "loss": 0.0, | |
| "reward": 0.20276049276435515, | |
| "reward_std": 0.26876165717840195, | |
| "rewards/cosine_scaled_reward": 0.0905617168173194, | |
| "rewards/format_reward": 0.5208333432674408, | |
| "step": 88 | |
| }, | |
| { | |
| "completion_length": 3106.9583740234375, | |
| "epoch": 0.10171428571428572, | |
| "grad_norm": 0.15616898238658905, | |
| "kl": 0.0005943775177001953, | |
| "learning_rate": 9.83423155058946e-07, | |
| "loss": 0.0, | |
| "reward": 0.14823118224740028, | |
| "reward_std": 0.25535859540104866, | |
| "rewards/cosine_scaled_reward": 0.07057950645685196, | |
| "rewards/format_reward": 0.3541666716337204, | |
| "step": 89 | |
| }, | |
| { | |
| "completion_length": 3029.5625610351562, | |
| "epoch": 0.10285714285714286, | |
| "grad_norm": 0.16313032805919647, | |
| "kl": 0.0008497238159179688, | |
| "learning_rate": 9.825677631722435e-07, | |
| "loss": 0.0, | |
| "reward": 0.10649214684963226, | |
| "reward_std": 0.3150200583040714, | |
| "rewards/cosine_scaled_reward": 0.01097804619348608, | |
| "rewards/format_reward": 0.3750000149011612, | |
| "step": 90 | |
| }, | |
| { | |
| "completion_length": 3242.4583740234375, | |
| "epoch": 0.104, | |
| "grad_norm": 0.1812453716993332, | |
| "kl": 0.0005056858062744141, | |
| "learning_rate": 9.816912885430258e-07, | |
| "loss": 0.0, | |
| "reward": -0.016676221042871475, | |
| "reward_std": 0.2188626378774643, | |
| "rewards/cosine_scaled_reward": -0.1111946739256382, | |
| "rewards/format_reward": 0.27083333395421505, | |
| "step": 91 | |
| }, | |
| { | |
| "completion_length": 2669.2709350585938, | |
| "epoch": 0.10514285714285715, | |
| "grad_norm": 0.1882462203502655, | |
| "kl": 0.0012416839599609375, | |
| "learning_rate": 9.807937738894303e-07, | |
| "loss": 0.0, | |
| "reward": 0.12547802366316319, | |
| "reward_std": 0.21191112510859966, | |
| "rewards/cosine_scaled_reward": 0.0004324503242969513, | |
| "rewards/format_reward": 0.47916667722165585, | |
| "step": 92 | |
| }, | |
| { | |
| "completion_length": 2442.0000610351562, | |
| "epoch": 0.10628571428571429, | |
| "grad_norm": 0.2229626625776291, | |
| "kl": 0.001369476318359375, | |
| "learning_rate": 9.798752629550546e-07, | |
| "loss": 0.0001, | |
| "reward": 0.15711925993673503, | |
| "reward_std": 0.2689987123012543, | |
| "rewards/cosine_scaled_reward": 0.030423451215028763, | |
| "rewards/format_reward": 0.5208333432674408, | |
| "step": 93 | |
| }, | |
| { | |
| "completion_length": 3294.2500610351562, | |
| "epoch": 0.10742857142857143, | |
| "grad_norm": 0.16960301995277405, | |
| "kl": 0.0009059906005859375, | |
| "learning_rate": 9.78935800506826e-07, | |
| "loss": 0.0, | |
| "reward": -0.04972950741648674, | |
| "reward_std": 0.20636767707765102, | |
| "rewards/cosine_scaled_reward": -0.11113300547003746, | |
| "rewards/format_reward": 0.14583333395421505, | |
| "step": 94 | |
| }, | |
| { | |
| "completion_length": 3159.791748046875, | |
| "epoch": 0.10857142857142857, | |
| "grad_norm": 0.14427055418491364, | |
| "kl": 0.00042057037353515625, | |
| "learning_rate": 9.779754323328192e-07, | |
| "loss": 0.0, | |
| "reward": -0.11810954473912716, | |
| "reward_std": 0.1571381390094757, | |
| "rewards/cosine_scaled_reward": -0.23719513043761253, | |
| "rewards/format_reward": 0.2500000037252903, | |
| "step": 95 | |
| }, | |
| { | |
| "completion_length": 2514.312530517578, | |
| "epoch": 0.10971428571428571, | |
| "grad_norm": 0.24939848482608795, | |
| "kl": 0.0015869140625, | |
| "learning_rate": 9.769942052400235e-07, | |
| "loss": 0.0001, | |
| "reward": 0.07279813662171364, | |
| "reward_std": 0.17102814465761185, | |
| "rewards/cosine_scaled_reward": -0.06484721228480339, | |
| "rewards/format_reward": 0.479166679084301, | |
| "step": 96 | |
| }, | |
| { | |
| "completion_length": 2968.104248046875, | |
| "epoch": 0.11085714285714286, | |
| "grad_norm": 0.2483038604259491, | |
| "kl": 0.0010707378387451172, | |
| "learning_rate": 9.759921670520634e-07, | |
| "loss": 0.0, | |
| "reward": 0.08805497021785413, | |
| "reward_std": 0.24433772545307875, | |
| "rewards/cosine_scaled_reward": -0.018354391679167747, | |
| "rewards/format_reward": 0.3958333395421505, | |
| "step": 97 | |
| }, | |
| { | |
| "completion_length": 2667.791717529297, | |
| "epoch": 0.112, | |
| "grad_norm": 0.20968908071517944, | |
| "kl": 0.0012552738189697266, | |
| "learning_rate": 9.749693666068663e-07, | |
| "loss": 0.0001, | |
| "reward": 0.050092536956071854, | |
| "reward_std": 0.19893871247768402, | |
| "rewards/cosine_scaled_reward": -0.0555708110332489, | |
| "rewards/format_reward": 0.35416667722165585, | |
| "step": 98 | |
| }, | |
| { | |
| "completion_length": 2831.875030517578, | |
| "epoch": 0.11314285714285714, | |
| "grad_norm": 0.20566846430301666, | |
| "kl": 0.0006589889526367188, | |
| "learning_rate": 9.739258537542835e-07, | |
| "loss": 0.0, | |
| "reward": 0.14856231398880482, | |
| "reward_std": 0.24846229702234268, | |
| "rewards/cosine_scaled_reward": 0.042768311919644475, | |
| "rewards/format_reward": 0.4375000223517418, | |
| "step": 99 | |
| }, | |
| { | |
| "completion_length": 3068.729248046875, | |
| "epoch": 0.11428571428571428, | |
| "grad_norm": 0.19074195623397827, | |
| "kl": 0.0006690025329589844, | |
| "learning_rate": 9.728616793536587e-07, | |
| "loss": 0.0, | |
| "reward": 0.1303813187405467, | |
| "reward_std": 0.2905452437698841, | |
| "rewards/cosine_scaled_reward": 0.04835892841219902, | |
| "rewards/format_reward": 0.3750000149011612, | |
| "step": 100 | |
| }, | |
| { | |
| "completion_length": 2701.229248046875, | |
| "epoch": 0.11542857142857142, | |
| "grad_norm": 0.22680342197418213, | |
| "kl": 0.00116729736328125, | |
| "learning_rate": 9.717768952713511e-07, | |
| "loss": 0.0, | |
| "reward": 0.10164168104529381, | |
| "reward_std": 0.29300505481660366, | |
| "rewards/cosine_scaled_reward": 0.005210310220718384, | |
| "rewards/format_reward": 0.375, | |
| "step": 101 | |
| }, | |
| { | |
| "completion_length": 2577.4168090820312, | |
| "epoch": 0.11657142857142858, | |
| "grad_norm": 0.16007407009601593, | |
| "kl": 0.0012121200561523438, | |
| "learning_rate": 9.706715543782064e-07, | |
| "loss": 0.0, | |
| "reward": 0.08228157926350832, | |
| "reward_std": 0.20707003772258759, | |
| "rewards/cosine_scaled_reward": -0.04624825902283192, | |
| "rewards/format_reward": 0.4583333432674408, | |
| "step": 102 | |
| }, | |
| { | |
| "completion_length": 2794.8541870117188, | |
| "epoch": 0.11771428571428572, | |
| "grad_norm": 0.17102256417274475, | |
| "kl": 0.0010852813720703125, | |
| "learning_rate": 9.695457105469804e-07, | |
| "loss": 0.0, | |
| "reward": 0.08034474775195122, | |
| "reward_std": 0.23851236328482628, | |
| "rewards/cosine_scaled_reward": -0.034777700901031494, | |
| "rewards/format_reward": 0.416666679084301, | |
| "step": 103 | |
| }, | |
| { | |
| "completion_length": 2514.0626220703125, | |
| "epoch": 0.11885714285714286, | |
| "grad_norm": 0.18136081099510193, | |
| "kl": 0.0005865097045898438, | |
| "learning_rate": 9.683994186497132e-07, | |
| "loss": 0.0, | |
| "reward": 0.25316645577549934, | |
| "reward_std": 0.23620141111314297, | |
| "rewards/cosine_scaled_reward": 0.16903768852353096, | |
| "rewards/format_reward": 0.4791666716337204, | |
| "step": 104 | |
| }, | |
| { | |
| "completion_length": 2782.4584350585938, | |
| "epoch": 0.12, | |
| "grad_norm": 0.20108939707279205, | |
| "kl": 0.001331329345703125, | |
| "learning_rate": 9.672327345550543e-07, | |
| "loss": 0.0001, | |
| "reward": 0.046333261765539646, | |
| "reward_std": 0.253768190741539, | |
| "rewards/cosine_scaled_reward": -0.08485639095306396, | |
| "rewards/format_reward": 0.4166666716337204, | |
| "step": 105 | |
| }, | |
| { | |
| "completion_length": 2904.1251220703125, | |
| "epoch": 0.12114285714285715, | |
| "grad_norm": 0.19156429171562195, | |
| "kl": 0.0013818740844726562, | |
| "learning_rate": 9.66045715125541e-07, | |
| "loss": 0.0001, | |
| "reward": 0.15791430696845055, | |
| "reward_std": 0.28573495149612427, | |
| "rewards/cosine_scaled_reward": 0.0755169503390789, | |
| "rewards/format_reward": 0.3958333395421505, | |
| "step": 106 | |
| }, | |
| { | |
| "completion_length": 2129.6459045410156, | |
| "epoch": 0.12228571428571429, | |
| "grad_norm": 0.19605104625225067, | |
| "kl": 0.0010166168212890625, | |
| "learning_rate": 9.648384182148252e-07, | |
| "loss": 0.0, | |
| "reward": 0.0870693230535835, | |
| "reward_std": 0.3030298389494419, | |
| "rewards/cosine_scaled_reward": -0.10336481593549252, | |
| "rewards/format_reward": 0.6458333432674408, | |
| "step": 107 | |
| }, | |
| { | |
| "completion_length": 2147.6041870117188, | |
| "epoch": 0.12342857142857143, | |
| "grad_norm": 0.1981632560491562, | |
| "kl": 0.0015201568603515625, | |
| "learning_rate": 9.636109026648554e-07, | |
| "loss": 0.0001, | |
| "reward": 0.28709910926409066, | |
| "reward_std": 0.2012571543455124, | |
| "rewards/cosine_scaled_reward": 0.1342942900955677, | |
| "rewards/format_reward": 0.6875000149011612, | |
| "step": 108 | |
| }, | |
| { | |
| "completion_length": 3073.666748046875, | |
| "epoch": 0.12457142857142857, | |
| "grad_norm": 0.17615145444869995, | |
| "kl": 0.0011663436889648438, | |
| "learning_rate": 9.623632283030077e-07, | |
| "loss": 0.0, | |
| "reward": -0.021478969603776932, | |
| "reward_std": 0.189506895840168, | |
| "rewards/cosine_scaled_reward": -0.13986263051629066, | |
| "rewards/format_reward": 0.3333333358168602, | |
| "step": 109 | |
| }, | |
| { | |
| "completion_length": 2523.3125915527344, | |
| "epoch": 0.12571428571428572, | |
| "grad_norm": 0.19854386150836945, | |
| "kl": 0.0023508071899414062, | |
| "learning_rate": 9.610954559391704e-07, | |
| "loss": 0.0001, | |
| "reward": 0.4017367544583976, | |
| "reward_std": 0.22793768532574177, | |
| "rewards/cosine_scaled_reward": 0.2740056961774826, | |
| "rewards/format_reward": 0.7291666865348816, | |
| "step": 110 | |
| }, | |
| { | |
| "completion_length": 1766.6459197998047, | |
| "epoch": 0.12685714285714286, | |
| "grad_norm": 0.2702421545982361, | |
| "kl": 0.001590728759765625, | |
| "learning_rate": 9.598076473627796e-07, | |
| "loss": 0.0001, | |
| "reward": 0.251003984361887, | |
| "reward_std": 0.2790553644299507, | |
| "rewards/cosine_scaled_reward": 0.07046932214871049, | |
| "rewards/format_reward": 0.7500000149011612, | |
| "step": 111 | |
| }, | |
| { | |
| "completion_length": 3026.041748046875, | |
| "epoch": 0.128, | |
| "grad_norm": 0.1819557249546051, | |
| "kl": 0.0011844635009765625, | |
| "learning_rate": 9.58499865339809e-07, | |
| "loss": 0.0, | |
| "reward": 0.05430050566792488, | |
| "reward_std": 0.2979404255747795, | |
| "rewards/cosine_scaled_reward": -0.052672116085886955, | |
| "rewards/format_reward": 0.3750000074505806, | |
| "step": 112 | |
| }, | |
| { | |
| "completion_length": 2138.541717529297, | |
| "epoch": 0.12914285714285714, | |
| "grad_norm": 0.2193819135427475, | |
| "kl": 0.0021915435791015625, | |
| "learning_rate": 9.571721736097088e-07, | |
| "loss": 0.0001, | |
| "reward": 0.20246881432831287, | |
| "reward_std": 0.2825229614973068, | |
| "rewards/cosine_scaled_reward": 0.05767783522605896, | |
| "rewards/format_reward": 0.6041666697710752, | |
| "step": 113 | |
| }, | |
| { | |
| "completion_length": 1764.7083740234375, | |
| "epoch": 0.13028571428571428, | |
| "grad_norm": 0.21886524558067322, | |
| "kl": 0.001209259033203125, | |
| "learning_rate": 9.55824636882301e-07, | |
| "loss": 0.0, | |
| "reward": 0.2999228276312351, | |
| "reward_std": 0.28240448236465454, | |
| "rewards/cosine_scaled_reward": 0.13528712838888168, | |
| "rewards/format_reward": 0.7291666716337204, | |
| "step": 114 | |
| }, | |
| { | |
| "completion_length": 2939.8958740234375, | |
| "epoch": 0.13142857142857142, | |
| "grad_norm": 0.16435398161411285, | |
| "kl": 0.0017871856689453125, | |
| "learning_rate": 9.54457320834625e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1701457817107439, | |
| "reward_std": 0.26418206468224525, | |
| "rewards/cosine_scaled_reward": 0.06508506275713444, | |
| "rewards/format_reward": 0.4583333432674408, | |
| "step": 115 | |
| }, | |
| { | |
| "completion_length": 2467.791748046875, | |
| "epoch": 0.13257142857142856, | |
| "grad_norm": 0.21436309814453125, | |
| "kl": 0.004275321960449219, | |
| "learning_rate": 9.530702921077358e-07, | |
| "loss": 0.0002, | |
| "reward": 0.05696129985153675, | |
| "reward_std": 0.3307487964630127, | |
| "rewards/cosine_scaled_reward": -0.08862798800691962, | |
| "rewards/format_reward": 0.47916667722165585, | |
| "step": 116 | |
| }, | |
| { | |
| "completion_length": 3445.6875610351562, | |
| "epoch": 0.1337142857142857, | |
| "grad_norm": 0.14990511536598206, | |
| "kl": 0.0016841888427734375, | |
| "learning_rate": 9.516636183034564e-07, | |
| "loss": 0.0001, | |
| "reward": -0.1102401977404952, | |
| "reward_std": 0.20692161843180656, | |
| "rewards/cosine_scaled_reward": -0.18584159016609192, | |
| "rewards/format_reward": 0.12500000186264515, | |
| "step": 117 | |
| }, | |
| { | |
| "completion_length": 1913.9792175292969, | |
| "epoch": 0.13485714285714287, | |
| "grad_norm": 0.2434745579957962, | |
| "kl": 0.0030345916748046875, | |
| "learning_rate": 9.502373679810839e-07, | |
| "loss": 0.0001, | |
| "reward": 0.27154221711680293, | |
| "reward_std": 0.22354764118790627, | |
| "rewards/cosine_scaled_reward": 0.09904532507061958, | |
| "rewards/format_reward": 0.7500000149011612, | |
| "step": 118 | |
| }, | |
| { | |
| "completion_length": 1981.0833740234375, | |
| "epoch": 0.136, | |
| "grad_norm": 0.29008790850639343, | |
| "kl": 0.004032135009765625, | |
| "learning_rate": 9.487916106540465e-07, | |
| "loss": 0.0002, | |
| "reward": 0.03949427045881748, | |
| "reward_std": 0.11318811401724815, | |
| "rewards/cosine_scaled_reward": -0.14376981928944588, | |
| "rewards/format_reward": 0.5833333432674408, | |
| "step": 119 | |
| }, | |
| { | |
| "completion_length": 2934.166748046875, | |
| "epoch": 0.13714285714285715, | |
| "grad_norm": 0.22969283163547516, | |
| "kl": 0.0017414093017578125, | |
| "learning_rate": 9.473264167865171e-07, | |
| "loss": 0.0001, | |
| "reward": 0.04561608098447323, | |
| "reward_std": 0.4027886167168617, | |
| "rewards/cosine_scaled_reward": -0.09728806652128696, | |
| "rewards/format_reward": 0.4583333507180214, | |
| "step": 120 | |
| }, | |
| { | |
| "completion_length": 2774.750030517578, | |
| "epoch": 0.1382857142857143, | |
| "grad_norm": 0.1892664134502411, | |
| "kl": 0.0012845993041992188, | |
| "learning_rate": 9.458418577899774e-07, | |
| "loss": 0.0001, | |
| "reward": 0.10860130004584789, | |
| "reward_std": 0.23990362137556076, | |
| "rewards/cosine_scaled_reward": 0.002404959872364998, | |
| "rewards/format_reward": 0.41666667722165585, | |
| "step": 121 | |
| }, | |
| { | |
| "completion_length": 2878.8333740234375, | |
| "epoch": 0.13942857142857143, | |
| "grad_norm": 0.2213921993970871, | |
| "kl": 0.0037384033203125, | |
| "learning_rate": 9.443380060197385e-07, | |
| "loss": 0.0001, | |
| "reward": -0.07514401245862246, | |
| "reward_std": 0.19457928091287613, | |
| "rewards/cosine_scaled_reward": -0.1945231556892395, | |
| "rewards/format_reward": 0.29166667349636555, | |
| "step": 122 | |
| }, | |
| { | |
| "completion_length": 2914.5000610351562, | |
| "epoch": 0.14057142857142857, | |
| "grad_norm": 1.7548092603683472, | |
| "kl": 0.0517425537109375, | |
| "learning_rate": 9.428149347714143e-07, | |
| "loss": 0.0021, | |
| "reward": -0.013575404649600387, | |
| "reward_std": 0.22872992418706417, | |
| "rewards/cosine_scaled_reward": -0.1315056849271059, | |
| "rewards/format_reward": 0.3333333395421505, | |
| "step": 123 | |
| }, | |
| { | |
| "completion_length": 2563.5625915527344, | |
| "epoch": 0.1417142857142857, | |
| "grad_norm": 0.2274925261735916, | |
| "kl": 0.00177001953125, | |
| "learning_rate": 9.412727182773486e-07, | |
| "loss": 0.0001, | |
| "reward": 0.17911872267723083, | |
| "reward_std": 0.3024190813302994, | |
| "rewards/cosine_scaled_reward": 0.028859375044703484, | |
| "rewards/format_reward": 0.6041666865348816, | |
| "step": 124 | |
| }, | |
| { | |
| "completion_length": 2916.166748046875, | |
| "epoch": 0.14285714285714285, | |
| "grad_norm": 0.19302645325660706, | |
| "kl": 0.0012264251708984375, | |
| "learning_rate": 9.397114317029974e-07, | |
| "loss": 0.0, | |
| "reward": 0.04521351866424084, | |
| "reward_std": 0.183570247143507, | |
| "rewards/cosine_scaled_reward": -0.0881996639072895, | |
| "rewards/format_reward": 0.4375000111758709, | |
| "step": 125 | |
| }, | |
| { | |
| "completion_length": 2168.3958740234375, | |
| "epoch": 0.144, | |
| "grad_norm": 0.3510819375514984, | |
| "kl": 0.0034084320068359375, | |
| "learning_rate": 9.381311511432658e-07, | |
| "loss": 0.0001, | |
| "reward": 0.07385214976966381, | |
| "reward_std": 0.2289694994688034, | |
| "rewards/cosine_scaled_reward": -0.10627560317516327, | |
| "rewards/format_reward": 0.6041666716337204, | |
| "step": 126 | |
| }, | |
| { | |
| "completion_length": 2812.9583740234375, | |
| "epoch": 0.14514285714285713, | |
| "grad_norm": 0.1580205112695694, | |
| "kl": 0.0012226104736328125, | |
| "learning_rate": 9.36531953618799e-07, | |
| "loss": 0.0, | |
| "reward": 0.07576266676187515, | |
| "reward_std": 0.22248199209570885, | |
| "rewards/cosine_scaled_reward": -0.026177331805229187, | |
| "rewards/format_reward": 0.37500000558793545, | |
| "step": 127 | |
| }, | |
| { | |
| "completion_length": 2719.9583435058594, | |
| "epoch": 0.1462857142857143, | |
| "grad_norm": 0.1932969093322754, | |
| "kl": 0.001880645751953125, | |
| "learning_rate": 9.34913917072228e-07, | |
| "loss": 0.0001, | |
| "reward": 0.12211918644607067, | |
| "reward_std": 0.22235066816210747, | |
| "rewards/cosine_scaled_reward": 0.022739887237548828, | |
| "rewards/format_reward": 0.3958333395421505, | |
| "step": 128 | |
| }, | |
| { | |
| "completion_length": 1889.4375457763672, | |
| "epoch": 0.14742857142857144, | |
| "grad_norm": 0.27860602736473083, | |
| "kl": 0.00513458251953125, | |
| "learning_rate": 9.332771203643714e-07, | |
| "loss": 0.0002, | |
| "reward": 0.191393559332937, | |
| "reward_std": 0.29455113410949707, | |
| "rewards/cosine_scaled_reward": 0.012313703075051308, | |
| "rewards/format_reward": 0.6875000074505806, | |
| "step": 129 | |
| }, | |
| { | |
| "completion_length": 3178.979248046875, | |
| "epoch": 0.14857142857142858, | |
| "grad_norm": 0.1898622065782547, | |
| "kl": 0.0014190673828125, | |
| "learning_rate": 9.316216432703916e-07, | |
| "loss": 0.0001, | |
| "reward": -0.03939210996031761, | |
| "reward_std": 0.2819094732403755, | |
| "rewards/cosine_scaled_reward": -0.1552355084568262, | |
| "rewards/format_reward": 0.3125000111758709, | |
| "step": 130 | |
| }, | |
| { | |
| "completion_length": 2920.6875610351562, | |
| "epoch": 0.14971428571428572, | |
| "grad_norm": 0.21076896786689758, | |
| "kl": 0.00131988525390625, | |
| "learning_rate": 9.299475664759068e-07, | |
| "loss": 0.0001, | |
| "reward": 0.07247342355549335, | |
| "reward_std": 0.2505417540669441, | |
| "rewards/cosine_scaled_reward": -0.04478706605732441, | |
| "rewards/format_reward": 0.4166666828095913, | |
| "step": 131 | |
| }, | |
| { | |
| "completion_length": 2784.8750610351562, | |
| "epoch": 0.15085714285714286, | |
| "grad_norm": 0.19709761440753937, | |
| "kl": 0.0018377304077148438, | |
| "learning_rate": 9.282549715730579e-07, | |
| "loss": 0.0001, | |
| "reward": 0.018537950702011585, | |
| "reward_std": 0.27707668393850327, | |
| "rewards/cosine_scaled_reward": -0.10952132218517363, | |
| "rewards/format_reward": 0.3958333395421505, | |
| "step": 132 | |
| }, | |
| { | |
| "completion_length": 2956.979248046875, | |
| "epoch": 0.152, | |
| "grad_norm": 0.23451754450798035, | |
| "kl": 0.002567291259765625, | |
| "learning_rate": 9.265439410565328e-07, | |
| "loss": 0.0001, | |
| "reward": -0.04261648189276457, | |
| "reward_std": 0.2717365212738514, | |
| "rewards/cosine_scaled_reward": -0.18468540161848068, | |
| "rewards/format_reward": 0.3958333544433117, | |
| "step": 133 | |
| }, | |
| { | |
| "completion_length": 1903.354232788086, | |
| "epoch": 0.15314285714285714, | |
| "grad_norm": 0.30252185463905334, | |
| "kl": 0.0040454864501953125, | |
| "learning_rate": 9.248145583195447e-07, | |
| "loss": 0.0002, | |
| "reward": 0.1737231626175344, | |
| "reward_std": 0.2514979690313339, | |
| "rewards/cosine_scaled_reward": -0.008977729827165604, | |
| "rewards/format_reward": 0.6875000149011612, | |
| "step": 134 | |
| }, | |
| { | |
| "completion_length": 2347.416778564453, | |
| "epoch": 0.15428571428571428, | |
| "grad_norm": 0.1927800178527832, | |
| "kl": 0.001873016357421875, | |
| "learning_rate": 9.230669076497687e-07, | |
| "loss": 0.0001, | |
| "reward": 0.09690875932574272, | |
| "reward_std": 0.20963543467223644, | |
| "rewards/cosine_scaled_reward": -0.05220409855246544, | |
| "rewards/format_reward": 0.5208333432674408, | |
| "step": 135 | |
| }, | |
| { | |
| "completion_length": 2465.5834350585938, | |
| "epoch": 0.15542857142857142, | |
| "grad_norm": 0.29496052861213684, | |
| "kl": 0.002719879150390625, | |
| "learning_rate": 9.213010742252327e-07, | |
| "loss": 0.0001, | |
| "reward": 0.04496379243209958, | |
| "reward_std": 0.30501750111579895, | |
| "rewards/cosine_scaled_reward": -0.12493768334388733, | |
| "rewards/format_reward": 0.5416666828095913, | |
| "step": 136 | |
| }, | |
| { | |
| "completion_length": 2833.5000610351562, | |
| "epoch": 0.15657142857142858, | |
| "grad_norm": 0.20566634833812714, | |
| "kl": 0.003387451171875, | |
| "learning_rate": 9.195171441101668e-07, | |
| "loss": 0.0001, | |
| "reward": -0.012183592654764652, | |
| "reward_std": 0.2676853369921446, | |
| "rewards/cosine_scaled_reward": -0.12493430450558662, | |
| "rewards/format_reward": 0.31250002048909664, | |
| "step": 137 | |
| }, | |
| { | |
| "completion_length": 2088.8125610351562, | |
| "epoch": 0.15771428571428572, | |
| "grad_norm": 0.2401277720928192, | |
| "kl": 0.0029354095458984375, | |
| "learning_rate": 9.177152042508077e-07, | |
| "loss": 0.0001, | |
| "reward": 0.31860150722786784, | |
| "reward_std": 0.2921822927892208, | |
| "rewards/cosine_scaled_reward": 0.1930521186441183, | |
| "rewards/format_reward": 0.6458333432674408, | |
| "step": 138 | |
| }, | |
| { | |
| "completion_length": 2617.1251220703125, | |
| "epoch": 0.15885714285714286, | |
| "grad_norm": 0.19433732330799103, | |
| "kl": 0.0014438629150390625, | |
| "learning_rate": 9.158953424711624e-07, | |
| "loss": 0.0001, | |
| "reward": 0.09779603406786919, | |
| "reward_std": 0.28393205627799034, | |
| "rewards/cosine_scaled_reward": -0.0396699383854866, | |
| "rewards/format_reward": 0.47916667349636555, | |
| "step": 139 | |
| }, | |
| { | |
| "completion_length": 2479.6876220703125, | |
| "epoch": 0.16, | |
| "grad_norm": 0.2356812208890915, | |
| "kl": 0.00270843505859375, | |
| "learning_rate": 9.140576474687263e-07, | |
| "loss": 0.0001, | |
| "reward": 0.16233161371201277, | |
| "reward_std": 0.21454461477696896, | |
| "rewards/cosine_scaled_reward": 0.020634490996599197, | |
| "rewards/format_reward": 0.5625000149011612, | |
| "step": 140 | |
| }, | |
| { | |
| "completion_length": 2655.8125, | |
| "epoch": 0.16114285714285714, | |
| "grad_norm": 0.21176272630691528, | |
| "kl": 0.003421783447265625, | |
| "learning_rate": 9.122022088101613e-07, | |
| "loss": 0.0001, | |
| "reward": 0.14447204023599625, | |
| "reward_std": 0.23380305618047714, | |
| "rewards/cosine_scaled_reward": 0.04617850482463837, | |
| "rewards/format_reward": 0.4166666865348816, | |
| "step": 141 | |
| }, | |
| { | |
| "completion_length": 2357.5000610351562, | |
| "epoch": 0.16228571428571428, | |
| "grad_norm": 0.23890313506126404, | |
| "kl": 0.0028333663940429688, | |
| "learning_rate": 9.103291169269299e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1481823343783617, | |
| "reward_std": 0.2089283987879753, | |
| "rewards/cosine_scaled_reward": 0.023761034943163395, | |
| "rewards/format_reward": 0.5000000149011612, | |
| "step": 142 | |
| }, | |
| { | |
| "completion_length": 2750.7500610351562, | |
| "epoch": 0.16342857142857142, | |
| "grad_norm": 0.17377115786075592, | |
| "kl": 0.0030498504638671875, | |
| "learning_rate": 9.084384631108882e-07, | |
| "loss": 0.0001, | |
| "reward": 0.19583595544099808, | |
| "reward_std": 0.2811162993311882, | |
| "rewards/cosine_scaled_reward": 0.0775480642914772, | |
| "rewards/format_reward": 0.5208333432674408, | |
| "step": 143 | |
| }, | |
| { | |
| "completion_length": 2684.3334350585938, | |
| "epoch": 0.16457142857142856, | |
| "grad_norm": 0.17148694396018982, | |
| "kl": 0.0034637451171875, | |
| "learning_rate": 9.065303395098358e-07, | |
| "loss": 0.0001, | |
| "reward": 0.18883249908685684, | |
| "reward_std": 0.3707999885082245, | |
| "rewards/cosine_scaled_reward": 0.0525783970952034, | |
| "rewards/format_reward": 0.5625000260770321, | |
| "step": 144 | |
| }, | |
| { | |
| "completion_length": 2799.6876220703125, | |
| "epoch": 0.1657142857142857, | |
| "grad_norm": 0.25570377707481384, | |
| "kl": 0.00708770751953125, | |
| "learning_rate": 9.046048391230247e-07, | |
| "loss": 0.0003, | |
| "reward": 0.08236825000494719, | |
| "reward_std": 0.2996014729142189, | |
| "rewards/cosine_scaled_reward": -0.02843803663563449, | |
| "rewards/format_reward": 0.3958333507180214, | |
| "step": 145 | |
| }, | |
| { | |
| "completion_length": 2574.7083740234375, | |
| "epoch": 0.16685714285714287, | |
| "grad_norm": 0.19306336343288422, | |
| "kl": 0.003398895263671875, | |
| "learning_rate": 9.026620557966279e-07, | |
| "loss": 0.0001, | |
| "reward": -0.035824403166770935, | |
| "reward_std": 0.19786337018013, | |
| "rewards/cosine_scaled_reward": -0.23294785246253014, | |
| "rewards/format_reward": 0.5625000223517418, | |
| "step": 146 | |
| }, | |
| { | |
| "completion_length": 2272.7709045410156, | |
| "epoch": 0.168, | |
| "grad_norm": 0.21024219691753387, | |
| "kl": 0.0027141571044921875, | |
| "learning_rate": 9.007020842191634e-07, | |
| "loss": 0.0001, | |
| "reward": 0.16781320981681347, | |
| "reward_std": 0.19364098459482193, | |
| "rewards/cosine_scaled_reward": 0.029284225136507303, | |
| "rewards/format_reward": 0.5625000074505806, | |
| "step": 147 | |
| }, | |
| { | |
| "completion_length": 2210.791717529297, | |
| "epoch": 0.16914285714285715, | |
| "grad_norm": 0.3196452856063843, | |
| "kl": 0.004055023193359375, | |
| "learning_rate": 8.987250199168808e-07, | |
| "loss": 0.0002, | |
| "reward": -0.05866700294427574, | |
| "reward_std": 0.2012275978922844, | |
| "rewards/cosine_scaled_reward": -0.26218296214938164, | |
| "rewards/format_reward": 0.5625000111758709, | |
| "step": 148 | |
| }, | |
| { | |
| "completion_length": 2778.4375610351562, | |
| "epoch": 0.1702857142857143, | |
| "grad_norm": 0.19716207683086395, | |
| "kl": 0.0034618377685546875, | |
| "learning_rate": 8.967309592491052e-07, | |
| "loss": 0.0001, | |
| "reward": 0.11828946322202682, | |
| "reward_std": 0.3204687051475048, | |
| "rewards/cosine_scaled_reward": 0.00033649057149887085, | |
| "rewards/format_reward": 0.45833334885537624, | |
| "step": 149 | |
| }, | |
| { | |
| "completion_length": 2391.8125610351562, | |
| "epoch": 0.17142857142857143, | |
| "grad_norm": 0.19943048059940338, | |
| "kl": 0.004062652587890625, | |
| "learning_rate": 8.9471999940354e-07, | |
| "loss": 0.0002, | |
| "reward": -0.03545256704092026, | |
| "reward_std": 0.1607068907469511, | |
| "rewards/cosine_scaled_reward": -0.21197712421417236, | |
| "rewards/format_reward": 0.5000000204890966, | |
| "step": 150 | |
| }, | |
| { | |
| "completion_length": 2911.4583740234375, | |
| "epoch": 0.17257142857142857, | |
| "grad_norm": 0.16765278577804565, | |
| "kl": 0.0033435821533203125, | |
| "learning_rate": 8.926922383915315e-07, | |
| "loss": 0.0001, | |
| "reward": 0.005968112964183092, | |
| "reward_std": 0.13167849741876125, | |
| "rewards/cosine_scaled_reward": -0.11089767143130302, | |
| "rewards/format_reward": 0.35416667722165585, | |
| "step": 151 | |
| }, | |
| { | |
| "completion_length": 3065.5834350585938, | |
| "epoch": 0.1737142857142857, | |
| "grad_norm": 0.17622444033622742, | |
| "kl": 0.006145477294921875, | |
| "learning_rate": 8.906477750432903e-07, | |
| "loss": 0.0002, | |
| "reward": 0.008522989228367805, | |
| "reward_std": 0.2847393900156021, | |
| "rewards/cosine_scaled_reward": -0.07492140308022499, | |
| "rewards/format_reward": 0.25000000186264515, | |
| "step": 152 | |
| }, | |
| { | |
| "completion_length": 2494.6459045410156, | |
| "epoch": 0.17485714285714285, | |
| "grad_norm": 0.21263031661510468, | |
| "kl": 0.0029783248901367188, | |
| "learning_rate": 8.88586709003076e-07, | |
| "loss": 0.0001, | |
| "reward": 0.17565679177641869, | |
| "reward_std": 0.28186945989727974, | |
| "rewards/cosine_scaled_reward": 0.05296103097498417, | |
| "rewards/format_reward": 0.5208333432674408, | |
| "step": 153 | |
| }, | |
| { | |
| "completion_length": 2867.6459045410156, | |
| "epoch": 0.176, | |
| "grad_norm": 0.16246166825294495, | |
| "kl": 0.0023956298828125, | |
| "learning_rate": 8.865091407243394e-07, | |
| "loss": 0.0001, | |
| "reward": 0.29472967982292175, | |
| "reward_std": 0.23086620680987835, | |
| "rewards/cosine_scaled_reward": 0.21928169950842857, | |
| "rewards/format_reward": 0.4791666716337204, | |
| "step": 154 | |
| }, | |
| { | |
| "completion_length": 3258.6250610351562, | |
| "epoch": 0.17714285714285713, | |
| "grad_norm": 0.24034130573272705, | |
| "kl": 0.0034580230712890625, | |
| "learning_rate": 8.844151714648274e-07, | |
| "loss": 0.0001, | |
| "reward": -0.12162286974489689, | |
| "reward_std": 0.20457392558455467, | |
| "rewards/cosine_scaled_reward": -0.2198924496769905, | |
| "rewards/format_reward": 0.18750000558793545, | |
| "step": 155 | |
| }, | |
| { | |
| "completion_length": 2978.7708740234375, | |
| "epoch": 0.1782857142857143, | |
| "grad_norm": 0.17035257816314697, | |
| "kl": 0.0047149658203125, | |
| "learning_rate": 8.823049032816478e-07, | |
| "loss": 0.0002, | |
| "reward": -0.09747781977057457, | |
| "reward_std": 0.16208519786596298, | |
| "rewards/cosine_scaled_reward": -0.21525413170456886, | |
| "rewards/format_reward": 0.2708333432674408, | |
| "step": 156 | |
| }, | |
| { | |
| "completion_length": 2397.541717529297, | |
| "epoch": 0.17942857142857144, | |
| "grad_norm": 0.22931884229183197, | |
| "kl": 0.0050029754638671875, | |
| "learning_rate": 8.801784390262943e-07, | |
| "loss": 0.0002, | |
| "reward": 0.11597155407071114, | |
| "reward_std": 0.28428420424461365, | |
| "rewards/cosine_scaled_reward": -0.011876031756401062, | |
| "rewards/format_reward": 0.47916667722165585, | |
| "step": 157 | |
| }, | |
| { | |
| "completion_length": 3140.5625610351562, | |
| "epoch": 0.18057142857142858, | |
| "grad_norm": 0.19685107469558716, | |
| "kl": 0.00397491455078125, | |
| "learning_rate": 8.780358823396352e-07, | |
| "loss": 0.0002, | |
| "reward": -0.08563036471605301, | |
| "reward_std": 0.16759349219501019, | |
| "rewards/cosine_scaled_reward": -0.19394809566438198, | |
| "rewards/format_reward": 0.25, | |
| "step": 158 | |
| }, | |
| { | |
| "completion_length": 2392.500030517578, | |
| "epoch": 0.18171428571428572, | |
| "grad_norm": 0.21569089591503143, | |
| "kl": 0.0052642822265625, | |
| "learning_rate": 8.758773376468604e-07, | |
| "loss": 0.0002, | |
| "reward": 0.3384667945356341, | |
| "reward_std": 0.27638140693306923, | |
| "rewards/cosine_scaled_reward": 0.17951813712716103, | |
| "rewards/format_reward": 0.7708333432674408, | |
| "step": 159 | |
| }, | |
| { | |
| "completion_length": 3030.6458740234375, | |
| "epoch": 0.18285714285714286, | |
| "grad_norm": 0.2277865707874298, | |
| "kl": 0.004344940185546875, | |
| "learning_rate": 8.737029101523929e-07, | |
| "loss": 0.0002, | |
| "reward": 0.20380683988332748, | |
| "reward_std": 0.2774137519299984, | |
| "rewards/cosine_scaled_reward": 0.09686106536537409, | |
| "rewards/format_reward": 0.5000000149011612, | |
| "step": 160 | |
| }, | |
| { | |
| "completion_length": 3296.9791870117188, | |
| "epoch": 0.184, | |
| "grad_norm": 0.15664881467819214, | |
| "kl": 0.0045928955078125, | |
| "learning_rate": 8.715127058347614e-07, | |
| "loss": 0.0002, | |
| "reward": -0.07018525979947299, | |
| "reward_std": 0.25186169147491455, | |
| "rewards/cosine_scaled_reward": -0.17596752382814884, | |
| "rewards/format_reward": 0.2500000074505806, | |
| "step": 161 | |
| }, | |
| { | |
| "completion_length": 3012.541748046875, | |
| "epoch": 0.18514285714285714, | |
| "grad_norm": 0.194331094622612, | |
| "kl": 0.0036983489990234375, | |
| "learning_rate": 8.693068314414344e-07, | |
| "loss": 0.0001, | |
| "reward": 0.06777064688503742, | |
| "reward_std": 0.35786426812410355, | |
| "rewards/cosine_scaled_reward": -0.06025504320859909, | |
| "rewards/format_reward": 0.4375000149011612, | |
| "step": 162 | |
| }, | |
| { | |
| "completion_length": 2512.3750915527344, | |
| "epoch": 0.18628571428571428, | |
| "grad_norm": 0.277127206325531, | |
| "kl": 0.00452423095703125, | |
| "learning_rate": 8.670853944836176e-07, | |
| "loss": 0.0002, | |
| "reward": 0.026186905801296234, | |
| "reward_std": 0.3294079564511776, | |
| "rewards/cosine_scaled_reward": -0.11063731834292412, | |
| "rewards/format_reward": 0.4375000111758709, | |
| "step": 163 | |
| }, | |
| { | |
| "completion_length": 2525.187530517578, | |
| "epoch": 0.18742857142857142, | |
| "grad_norm": 0.3039884567260742, | |
| "kl": 0.00612640380859375, | |
| "learning_rate": 8.648485032310144e-07, | |
| "loss": 0.0002, | |
| "reward": -0.005198385566473007, | |
| "reward_std": 0.21642658859491348, | |
| "rewards/cosine_scaled_reward": -0.1315268948674202, | |
| "rewards/format_reward": 0.3750000149011612, | |
| "step": 164 | |
| }, | |
| { | |
| "completion_length": 2689.1459350585938, | |
| "epoch": 0.18857142857142858, | |
| "grad_norm": 0.18826548755168915, | |
| "kl": 0.00409698486328125, | |
| "learning_rate": 8.625962667065487e-07, | |
| "loss": 0.0002, | |
| "reward": 0.1363761046086438, | |
| "reward_std": 0.29318901523947716, | |
| "rewards/cosine_scaled_reward": 0.008730031549930573, | |
| "rewards/format_reward": 0.5000000223517418, | |
| "step": 165 | |
| }, | |
| { | |
| "completion_length": 2415.8125915527344, | |
| "epoch": 0.18971428571428572, | |
| "grad_norm": 0.2114183008670807, | |
| "kl": 0.00537872314453125, | |
| "learning_rate": 8.603287946810513e-07, | |
| "loss": 0.0002, | |
| "reward": 0.01206995639950037, | |
| "reward_std": 0.1482153832912445, | |
| "rewards/cosine_scaled_reward": -0.14565513283014297, | |
| "rewards/format_reward": 0.47916668467223644, | |
| "step": 166 | |
| }, | |
| { | |
| "completion_length": 2857.7084350585938, | |
| "epoch": 0.19085714285714286, | |
| "grad_norm": 0.16645163297653198, | |
| "kl": 0.0044403076171875, | |
| "learning_rate": 8.580461976679099e-07, | |
| "loss": 0.0002, | |
| "reward": -0.04430033452808857, | |
| "reward_std": 0.2540858667343855, | |
| "rewards/cosine_scaled_reward": -0.18336289003491402, | |
| "rewards/format_reward": 0.37500000186264515, | |
| "step": 167 | |
| }, | |
| { | |
| "completion_length": 2432.104248046875, | |
| "epoch": 0.192, | |
| "grad_norm": 0.18748867511749268, | |
| "kl": 0.00420379638671875, | |
| "learning_rate": 8.557485869176825e-07, | |
| "loss": 0.0002, | |
| "reward": 0.10138805268798023, | |
| "reward_std": 0.2685610204935074, | |
| "rewards/cosine_scaled_reward": -0.07673629373311996, | |
| "rewards/format_reward": 0.6250000149011612, | |
| "step": 168 | |
| }, | |
| { | |
| "completion_length": 2429.7709045410156, | |
| "epoch": 0.19314285714285714, | |
| "grad_norm": 0.2035347819328308, | |
| "kl": 0.0036020278930664062, | |
| "learning_rate": 8.534360744126753e-07, | |
| "loss": 0.0001, | |
| "reward": 0.4033849276602268, | |
| "reward_std": 0.3233870640397072, | |
| "rewards/cosine_scaled_reward": 0.28906378895044327, | |
| "rewards/format_reward": 0.6875000149011612, | |
| "step": 169 | |
| }, | |
| { | |
| "completion_length": 2623.6250610351562, | |
| "epoch": 0.19428571428571428, | |
| "grad_norm": 0.18528129160404205, | |
| "kl": 0.004909515380859375, | |
| "learning_rate": 8.511087728614862e-07, | |
| "loss": 0.0002, | |
| "reward": 0.061040711123496294, | |
| "reward_std": 0.23592132702469826, | |
| "rewards/cosine_scaled_reward": -0.05934416502714157, | |
| "rewards/format_reward": 0.4166666716337204, | |
| "step": 170 | |
| }, | |
| { | |
| "completion_length": 2023.8750457763672, | |
| "epoch": 0.19542857142857142, | |
| "grad_norm": 0.2365913987159729, | |
| "kl": 0.004756927490234375, | |
| "learning_rate": 8.487667956935087e-07, | |
| "loss": 0.0002, | |
| "reward": 0.08947444148361683, | |
| "reward_std": 0.19351089000701904, | |
| "rewards/cosine_scaled_reward": -0.08239659294486046, | |
| "rewards/format_reward": 0.5833333432674408, | |
| "step": 171 | |
| }, | |
| { | |
| "completion_length": 2301.8125610351562, | |
| "epoch": 0.19657142857142856, | |
| "grad_norm": 0.23605789244174957, | |
| "kl": 0.00450897216796875, | |
| "learning_rate": 8.464102570534061e-07, | |
| "loss": 0.0002, | |
| "reward": 0.19074868597090244, | |
| "reward_std": 0.3016255050897598, | |
| "rewards/cosine_scaled_reward": 0.04505336657166481, | |
| "rewards/format_reward": 0.583333333954215, | |
| "step": 172 | |
| }, | |
| { | |
| "completion_length": 2721.6459045410156, | |
| "epoch": 0.1977142857142857, | |
| "grad_norm": 0.24565771222114563, | |
| "kl": 0.0044689178466796875, | |
| "learning_rate": 8.440392717955475e-07, | |
| "loss": 0.0002, | |
| "reward": 0.10209671594202518, | |
| "reward_std": 0.20993922092020512, | |
| "rewards/cosine_scaled_reward": -0.0017293132841587067, | |
| "rewards/format_reward": 0.3958333432674408, | |
| "step": 173 | |
| }, | |
| { | |
| "completion_length": 2793.6250610351562, | |
| "epoch": 0.19885714285714284, | |
| "grad_norm": 0.214183509349823, | |
| "kl": 0.00710296630859375, | |
| "learning_rate": 8.416539554784089e-07, | |
| "loss": 0.0003, | |
| "reward": 0.0948445312678814, | |
| "reward_std": 0.22529449313879013, | |
| "rewards/cosine_scaled_reward": -0.006497794296592474, | |
| "rewards/format_reward": 0.3958333432674408, | |
| "step": 174 | |
| }, | |
| { | |
| "completion_length": 2957.0001220703125, | |
| "epoch": 0.2, | |
| "grad_norm": 0.21723315119743347, | |
| "kl": 0.0075054168701171875, | |
| "learning_rate": 8.392544243589427e-07, | |
| "loss": 0.0003, | |
| "reward": 0.16869353270158172, | |
| "reward_std": 0.274255458265543, | |
| "rewards/cosine_scaled_reward": 0.047592975199222565, | |
| "rewards/format_reward": 0.5000000074505806, | |
| "step": 175 | |
| }, | |
| { | |
| "completion_length": 2953.979248046875, | |
| "epoch": 0.20114285714285715, | |
| "grad_norm": 0.1722651869058609, | |
| "kl": 0.0059967041015625, | |
| "learning_rate": 8.368407953869103e-07, | |
| "loss": 0.0002, | |
| "reward": 0.13873707130551338, | |
| "reward_std": 0.29712219163775444, | |
| "rewards/cosine_scaled_reward": 0.04709775559604168, | |
| "rewards/format_reward": 0.39583334140479565, | |
| "step": 176 | |
| }, | |
| { | |
| "completion_length": 2266.5000610351562, | |
| "epoch": 0.2022857142857143, | |
| "grad_norm": 0.19328881800174713, | |
| "kl": 0.00374603271484375, | |
| "learning_rate": 8.344131861991828e-07, | |
| "loss": 0.0002, | |
| "reward": 0.06314084958285093, | |
| "reward_std": 0.22284647449851036, | |
| "rewards/cosine_scaled_reward": -0.12701627984642982, | |
| "rewards/format_reward": 0.625, | |
| "step": 177 | |
| }, | |
| { | |
| "completion_length": 2629.979248046875, | |
| "epoch": 0.20342857142857143, | |
| "grad_norm": 0.20043830573558807, | |
| "kl": 0.00550079345703125, | |
| "learning_rate": 8.319717151140072e-07, | |
| "loss": 0.0002, | |
| "reward": 0.24929668940603733, | |
| "reward_std": 0.2040734402835369, | |
| "rewards/cosine_scaled_reward": 0.14905891939997673, | |
| "rewards/format_reward": 0.520833358168602, | |
| "step": 178 | |
| }, | |
| { | |
| "completion_length": 2652.1875610351562, | |
| "epoch": 0.20457142857142857, | |
| "grad_norm": 0.1805621236562729, | |
| "kl": 0.005802154541015625, | |
| "learning_rate": 8.295165011252396e-07, | |
| "loss": 0.0002, | |
| "reward": 0.09193149022758007, | |
| "reward_std": 0.20314152538776398, | |
| "rewards/cosine_scaled_reward": -0.03507839888334274, | |
| "rewards/format_reward": 0.4583333432674408, | |
| "step": 179 | |
| }, | |
| { | |
| "completion_length": 2596.375, | |
| "epoch": 0.2057142857142857, | |
| "grad_norm": 0.249373197555542, | |
| "kl": 0.00714111328125, | |
| "learning_rate": 8.270476638965461e-07, | |
| "loss": 0.0003, | |
| "reward": 0.11438339785672724, | |
| "reward_std": 0.3229541666805744, | |
| "rewards/cosine_scaled_reward": -0.010897724889218807, | |
| "rewards/format_reward": 0.4791666753590107, | |
| "step": 180 | |
| }, | |
| { | |
| "completion_length": 3172.5625, | |
| "epoch": 0.20685714285714285, | |
| "grad_norm": 0.15354838967323303, | |
| "kl": 0.00362396240234375, | |
| "learning_rate": 8.245653237555705e-07, | |
| "loss": 0.0001, | |
| "reward": 0.14108785180724226, | |
| "reward_std": 0.25704972073435783, | |
| "rewards/cosine_scaled_reward": 0.046188078820705414, | |
| "rewards/format_reward": 0.4166666865348816, | |
| "step": 181 | |
| }, | |
| { | |
| "completion_length": 2754.1458435058594, | |
| "epoch": 0.208, | |
| "grad_norm": 0.20978248119354248, | |
| "kl": 0.006561279296875, | |
| "learning_rate": 8.220696016880687e-07, | |
| "loss": 0.0003, | |
| "reward": 0.08122844994068146, | |
| "reward_std": 0.3094419874250889, | |
| "rewards/cosine_scaled_reward": -0.03756958991289139, | |
| "rewards/format_reward": 0.41666666977107525, | |
| "step": 182 | |
| }, | |
| { | |
| "completion_length": 2732.2708740234375, | |
| "epoch": 0.20914285714285713, | |
| "grad_norm": 0.1716933399438858, | |
| "kl": 0.006378173828125, | |
| "learning_rate": 8.195606193320136e-07, | |
| "loss": 0.0003, | |
| "reward": -0.040448714047670364, | |
| "reward_std": 0.20272203534841537, | |
| "rewards/cosine_scaled_reward": -0.21793191507458687, | |
| "rewards/format_reward": 0.5, | |
| "step": 183 | |
| }, | |
| { | |
| "completion_length": 2339.041748046875, | |
| "epoch": 0.2102857142857143, | |
| "grad_norm": 0.2090311050415039, | |
| "kl": 0.004291534423828125, | |
| "learning_rate": 8.170384989716657e-07, | |
| "loss": 0.0002, | |
| "reward": 0.3101842775940895, | |
| "reward_std": 0.32960085570812225, | |
| "rewards/cosine_scaled_reward": 0.19724036287516356, | |
| "rewards/format_reward": 0.6041666679084301, | |
| "step": 184 | |
| }, | |
| { | |
| "completion_length": 3183.229248046875, | |
| "epoch": 0.21142857142857144, | |
| "grad_norm": 0.1650100201368332, | |
| "kl": 0.006046295166015625, | |
| "learning_rate": 8.145033635316128e-07, | |
| "loss": 0.0002, | |
| "reward": -0.016166038811206818, | |
| "reward_std": 0.183053657412529, | |
| "rewards/cosine_scaled_reward": -0.11014915257692337, | |
| "rewards/format_reward": 0.27083333395421505, | |
| "step": 185 | |
| }, | |
| { | |
| "completion_length": 2894.291717529297, | |
| "epoch": 0.21257142857142858, | |
| "grad_norm": 0.1902281790971756, | |
| "kl": 0.003826141357421875, | |
| "learning_rate": 8.119553365707802e-07, | |
| "loss": 0.0002, | |
| "reward": 0.037780286045745015, | |
| "reward_std": 0.23388531804084778, | |
| "rewards/cosine_scaled_reward": -0.09303178638219833, | |
| "rewards/format_reward": 0.4166666716337204, | |
| "step": 186 | |
| }, | |
| { | |
| "completion_length": 2756.3959350585938, | |
| "epoch": 0.21371428571428572, | |
| "grad_norm": 0.19003655016422272, | |
| "kl": 0.0063323974609375, | |
| "learning_rate": 8.093945422764069e-07, | |
| "loss": 0.0003, | |
| "reward": 0.23986343410797417, | |
| "reward_std": 0.294619157910347, | |
| "rewards/cosine_scaled_reward": 0.1449273396283388, | |
| "rewards/format_reward": 0.5, | |
| "step": 187 | |
| }, | |
| { | |
| "completion_length": 2735.4166870117188, | |
| "epoch": 0.21485714285714286, | |
| "grad_norm": 0.20645277202129364, | |
| "kl": 0.006011962890625, | |
| "learning_rate": 8.068211054579943e-07, | |
| "loss": 0.0002, | |
| "reward": 0.18498247489333153, | |
| "reward_std": 0.38456569984555244, | |
| "rewards/cosine_scaled_reward": 0.05256199091672897, | |
| "rewards/format_reward": 0.5625000149011612, | |
| "step": 188 | |
| }, | |
| { | |
| "completion_length": 2607.7916870117188, | |
| "epoch": 0.216, | |
| "grad_norm": 0.20103368163108826, | |
| "kl": 0.009521484375, | |
| "learning_rate": 8.04235151541222e-07, | |
| "loss": 0.0004, | |
| "reward": 0.032526164315640926, | |
| "reward_std": 0.23300225287675858, | |
| "rewards/cosine_scaled_reward": -0.11186909675598145, | |
| "rewards/format_reward": 0.4583333395421505, | |
| "step": 189 | |
| }, | |
| { | |
| "completion_length": 2812.5833740234375, | |
| "epoch": 0.21714285714285714, | |
| "grad_norm": 0.1999751627445221, | |
| "kl": 0.00589752197265625, | |
| "learning_rate": 8.01636806561836e-07, | |
| "loss": 0.0002, | |
| "reward": 0.0322457030415535, | |
| "reward_std": 0.20498248934745789, | |
| "rewards/cosine_scaled_reward": -0.07093745563179255, | |
| "rewards/format_reward": 0.3333333432674408, | |
| "step": 190 | |
| }, | |
| { | |
| "completion_length": 3061.2709350585938, | |
| "epoch": 0.21828571428571428, | |
| "grad_norm": 0.24069836735725403, | |
| "kl": 0.008758544921875, | |
| "learning_rate": 7.990261971595048e-07, | |
| "loss": 0.0004, | |
| "reward": 0.003465789370238781, | |
| "reward_std": 0.2975287064909935, | |
| "rewards/cosine_scaled_reward": -0.10922680050134659, | |
| "rewards/format_reward": 0.3333333544433117, | |
| "step": 191 | |
| }, | |
| { | |
| "completion_length": 3110.6458740234375, | |
| "epoch": 0.21942857142857142, | |
| "grad_norm": 0.18123462796211243, | |
| "kl": 0.0098114013671875, | |
| "learning_rate": 7.964034505716476e-07, | |
| "loss": 0.0004, | |
| "reward": -0.00519880885258317, | |
| "reward_std": 0.23952285945415497, | |
| "rewards/cosine_scaled_reward": -0.09877765458077192, | |
| "rewards/format_reward": 0.27083333767950535, | |
| "step": 192 | |
| }, | |
| { | |
| "completion_length": 2421.7916870117188, | |
| "epoch": 0.22057142857142858, | |
| "grad_norm": 0.2296593189239502, | |
| "kl": 0.00974273681640625, | |
| "learning_rate": 7.93768694627233e-07, | |
| "loss": 0.0004, | |
| "reward": 0.07602208573371172, | |
| "reward_std": 0.19795489311218262, | |
| "rewards/cosine_scaled_reward": -0.09206605609506369, | |
| "rewards/format_reward": 0.5625000111758709, | |
| "step": 193 | |
| }, | |
| { | |
| "completion_length": 1688.0625610351562, | |
| "epoch": 0.22171428571428572, | |
| "grad_norm": 0.24463699758052826, | |
| "kl": 0.00638580322265625, | |
| "learning_rate": 7.911220577405484e-07, | |
| "loss": 0.0003, | |
| "reward": 0.46088459342718124, | |
| "reward_std": 0.39439669996500015, | |
| "rewards/cosine_scaled_reward": 0.29850273206830025, | |
| "rewards/format_reward": 0.8541666716337204, | |
| "step": 194 | |
| }, | |
| { | |
| "completion_length": 2056.791717529297, | |
| "epoch": 0.22285714285714286, | |
| "grad_norm": 0.18850582838058472, | |
| "kl": 0.007778167724609375, | |
| "learning_rate": 7.884636689049422e-07, | |
| "loss": 0.0003, | |
| "reward": 0.21396406181156635, | |
| "reward_std": 0.24946607649326324, | |
| "rewards/cosine_scaled_reward": 0.07859774492681026, | |
| "rewards/format_reward": 0.5833333358168602, | |
| "step": 195 | |
| }, | |
| { | |
| "completion_length": 3171.6875610351562, | |
| "epoch": 0.224, | |
| "grad_norm": 0.2034706175327301, | |
| "kl": 0.005645751953125, | |
| "learning_rate": 7.857936576865356e-07, | |
| "loss": 0.0002, | |
| "reward": -0.05634847842156887, | |
| "reward_std": 0.2312827706336975, | |
| "rewards/cosine_scaled_reward": -0.1469667460769415, | |
| "rewards/format_reward": 0.2291666679084301, | |
| "step": 196 | |
| }, | |
| { | |
| "completion_length": 2725.854217529297, | |
| "epoch": 0.22514285714285714, | |
| "grad_norm": 0.18927797675132751, | |
| "kl": 0.00467681884765625, | |
| "learning_rate": 7.831121542179086e-07, | |
| "loss": 0.0002, | |
| "reward": -0.00976499728858471, | |
| "reward_std": 0.3078451603651047, | |
| "rewards/cosine_scaled_reward": -0.14113588817417622, | |
| "rewards/format_reward": 0.37500001676380634, | |
| "step": 197 | |
| }, | |
| { | |
| "completion_length": 3027.479248046875, | |
| "epoch": 0.22628571428571428, | |
| "grad_norm": 0.21629135310649872, | |
| "kl": 0.0053081512451171875, | |
| "learning_rate": 7.804192891917571e-07, | |
| "loss": 0.0002, | |
| "reward": 0.06900226790457964, | |
| "reward_std": 0.2766474634408951, | |
| "rewards/cosine_scaled_reward": -0.037792665883898735, | |
| "rewards/format_reward": 0.3750000074505806, | |
| "step": 198 | |
| }, | |
| { | |
| "completion_length": 2622.0418090820312, | |
| "epoch": 0.22742857142857142, | |
| "grad_norm": 0.20776836574077606, | |
| "kl": 0.0056915283203125, | |
| "learning_rate": 7.777151938545235e-07, | |
| "loss": 0.0002, | |
| "reward": 0.15501756500452757, | |
| "reward_std": 0.3029539883136749, | |
| "rewards/cosine_scaled_reward": 0.03127571474760771, | |
| "rewards/format_reward": 0.5208333432674408, | |
| "step": 199 | |
| }, | |
| { | |
| "completion_length": 2262.979217529297, | |
| "epoch": 0.22857142857142856, | |
| "grad_norm": 0.222330704331398, | |
| "kl": 0.0066375732421875, | |
| "learning_rate": 7.75e-07, | |
| "loss": 0.0003, | |
| "reward": 0.26917505636811256, | |
| "reward_std": 0.32473092526197433, | |
| "rewards/cosine_scaled_reward": 0.12532378360629082, | |
| "rewards/format_reward": 0.645833333954215, | |
| "step": 200 | |
| }, | |
| { | |
| "completion_length": 2621.6666870117188, | |
| "epoch": 0.2297142857142857, | |
| "grad_norm": 0.19183361530303955, | |
| "kl": 0.006618499755859375, | |
| "learning_rate": 7.72273839962904e-07, | |
| "loss": 0.0003, | |
| "reward": 0.06654832325875759, | |
| "reward_std": 0.23418250493705273, | |
| "rewards/cosine_scaled_reward": -0.06533646397292614, | |
| "rewards/format_reward": 0.4583333395421505, | |
| "step": 201 | |
| }, | |
| { | |
| "completion_length": 2380.0, | |
| "epoch": 0.23085714285714284, | |
| "grad_norm": 0.21531449258327484, | |
| "kl": 0.0079803466796875, | |
| "learning_rate": 7.695368466124296e-07, | |
| "loss": 0.0003, | |
| "reward": 0.09720260743051767, | |
| "reward_std": 0.27152542024850845, | |
| "rewards/cosine_scaled_reward": -0.07546245865523815, | |
| "rewards/format_reward": 0.6041666865348816, | |
| "step": 202 | |
| }, | |
| { | |
| "completion_length": 2418.4375610351562, | |
| "epoch": 0.232, | |
| "grad_norm": 0.18618948757648468, | |
| "kl": 0.00667572021484375, | |
| "learning_rate": 7.667891533457718e-07, | |
| "loss": 0.0003, | |
| "reward": 0.14104736736044288, | |
| "reward_std": 0.23524043709039688, | |
| "rewards/cosine_scaled_reward": -0.011097889393568039, | |
| "rewards/format_reward": 0.5833333358168602, | |
| "step": 203 | |
| }, | |
| { | |
| "completion_length": 2163.5208435058594, | |
| "epoch": 0.23314285714285715, | |
| "grad_norm": 0.30891525745391846, | |
| "kl": 0.006256103515625, | |
| "learning_rate": 7.640308940816239e-07, | |
| "loss": 0.0003, | |
| "reward": 0.2278699278831482, | |
| "reward_std": 0.3605262115597725, | |
| "rewards/cosine_scaled_reward": 0.12358852848410606, | |
| "rewards/format_reward": 0.47916667722165585, | |
| "step": 204 | |
| }, | |
| { | |
| "completion_length": 2758.729217529297, | |
| "epoch": 0.2342857142857143, | |
| "grad_norm": 0.1707957684993744, | |
| "kl": 0.00688934326171875, | |
| "learning_rate": 7.612622032536507e-07, | |
| "loss": 0.0003, | |
| "reward": 0.13417839724570513, | |
| "reward_std": 0.3422238156199455, | |
| "rewards/cosine_scaled_reward": 0.006505465134978294, | |
| "rewards/format_reward": 0.5000000204890966, | |
| "step": 205 | |
| }, | |
| { | |
| "completion_length": 3193.3125610351562, | |
| "epoch": 0.23542857142857143, | |
| "grad_norm": 0.17781589925289154, | |
| "kl": 0.00615692138671875, | |
| "learning_rate": 7.584832158039378e-07, | |
| "loss": 0.0002, | |
| "reward": 0.015708591789007187, | |
| "reward_std": 0.2507881745696068, | |
| "rewards/cosine_scaled_reward": -0.09797392599284649, | |
| "rewards/format_reward": 0.3541666716337204, | |
| "step": 206 | |
| }, | |
| { | |
| "completion_length": 2658.0, | |
| "epoch": 0.23657142857142857, | |
| "grad_norm": 0.18779563903808594, | |
| "kl": 0.00539398193359375, | |
| "learning_rate": 7.556940671764124e-07, | |
| "loss": 0.0002, | |
| "reward": 0.14627259969711304, | |
| "reward_std": 0.3163585066795349, | |
| "rewards/cosine_scaled_reward": 0.029968852177262306, | |
| "rewards/format_reward": 0.4791666716337204, | |
| "step": 207 | |
| }, | |
| { | |
| "completion_length": 3152.1458740234375, | |
| "epoch": 0.2377142857142857, | |
| "grad_norm": 0.17349769175052643, | |
| "kl": 0.0095672607421875, | |
| "learning_rate": 7.528948933102438e-07, | |
| "loss": 0.0004, | |
| "reward": -0.13337488658726215, | |
| "reward_std": 0.16799047589302063, | |
| "rewards/cosine_scaled_reward": -0.24787230044603348, | |
| "rewards/format_reward": 0.229166679084301, | |
| "step": 208 | |
| }, | |
| { | |
| "completion_length": 2363.229217529297, | |
| "epoch": 0.23885714285714285, | |
| "grad_norm": 0.2527102828025818, | |
| "kl": 0.0085296630859375, | |
| "learning_rate": 7.500858306332172e-07, | |
| "loss": 0.0003, | |
| "reward": 0.13303392380475998, | |
| "reward_std": 0.22437193989753723, | |
| "rewards/cosine_scaled_reward": -0.0008901059627532959, | |
| "rewards/format_reward": 0.5, | |
| "step": 209 | |
| }, | |
| { | |
| "completion_length": 1666.6459045410156, | |
| "epoch": 0.24, | |
| "grad_norm": 0.2683139145374298, | |
| "kl": 0.006725311279296875, | |
| "learning_rate": 7.472670160550848e-07, | |
| "loss": 0.0003, | |
| "reward": 0.2863995414227247, | |
| "reward_std": 0.15318151377141476, | |
| "rewards/cosine_scaled_reward": 0.1167343258857727, | |
| "rewards/format_reward": 0.75, | |
| "step": 210 | |
| }, | |
| { | |
| "completion_length": 3162.1876220703125, | |
| "epoch": 0.24114285714285713, | |
| "grad_norm": 0.17517906427383423, | |
| "kl": 0.0062408447265625, | |
| "learning_rate": 7.444385869608921e-07, | |
| "loss": 0.0002, | |
| "reward": -0.13221720606088638, | |
| "reward_std": 0.19711453840136528, | |
| "rewards/cosine_scaled_reward": -0.2389248050749302, | |
| "rewards/format_reward": 0.2083333358168602, | |
| "step": 211 | |
| }, | |
| { | |
| "completion_length": 2667.0, | |
| "epoch": 0.2422857142857143, | |
| "grad_norm": 0.214364692568779, | |
| "kl": 0.0103759765625, | |
| "learning_rate": 7.416006812042827e-07, | |
| "loss": 0.0004, | |
| "reward": 0.144077368080616, | |
| "reward_std": 0.30216942727565765, | |
| "rewards/cosine_scaled_reward": 0.024308504071086645, | |
| "rewards/format_reward": 0.4791666828095913, | |
| "step": 212 | |
| }, | |
| { | |
| "completion_length": 3007.2291870117188, | |
| "epoch": 0.24342857142857144, | |
| "grad_norm": 0.18140621483325958, | |
| "kl": 0.00598907470703125, | |
| "learning_rate": 7.387534371007797e-07, | |
| "loss": 0.0002, | |
| "reward": -0.014454755932092667, | |
| "reward_std": 0.13768145814538002, | |
| "rewards/cosine_scaled_reward": -0.12324847280979156, | |
| "rewards/format_reward": 0.3125000111758709, | |
| "step": 213 | |
| }, | |
| { | |
| "completion_length": 2616.979248046875, | |
| "epoch": 0.24457142857142858, | |
| "grad_norm": 0.15950439870357513, | |
| "kl": 0.0040435791015625, | |
| "learning_rate": 7.358969934210438e-07, | |
| "loss": 0.0002, | |
| "reward": 0.09677460975944996, | |
| "reward_std": 0.24564701318740845, | |
| "rewards/cosine_scaled_reward": -0.07848542928695679, | |
| "rewards/format_reward": 0.604166679084301, | |
| "step": 214 | |
| }, | |
| { | |
| "completion_length": 3177.3126220703125, | |
| "epoch": 0.24571428571428572, | |
| "grad_norm": 0.1937507539987564, | |
| "kl": 0.0090789794921875, | |
| "learning_rate": 7.330314893841101e-07, | |
| "loss": 0.0004, | |
| "reward": -0.012199342250823975, | |
| "reward_std": 0.25811920315027237, | |
| "rewards/cosine_scaled_reward": -0.11902385950088501, | |
| "rewards/format_reward": 0.3125000111758709, | |
| "step": 215 | |
| }, | |
| { | |
| "completion_length": 2959.0000610351562, | |
| "epoch": 0.24685714285714286, | |
| "grad_norm": 0.17855827510356903, | |
| "kl": 0.0058441162109375, | |
| "learning_rate": 7.301570646506027e-07, | |
| "loss": 0.0002, | |
| "reward": -0.07763024140149355, | |
| "reward_std": 0.22440576925873756, | |
| "rewards/cosine_scaled_reward": -0.2029955154284835, | |
| "rewards/format_reward": 0.3125, | |
| "step": 216 | |
| }, | |
| { | |
| "completion_length": 2965.291748046875, | |
| "epoch": 0.248, | |
| "grad_norm": 0.1786404252052307, | |
| "kl": 0.0071868896484375, | |
| "learning_rate": 7.27273859315928e-07, | |
| "loss": 0.0003, | |
| "reward": 0.09465384157374501, | |
| "reward_std": 0.27161500602960587, | |
| "rewards/cosine_scaled_reward": -0.010866086289752275, | |
| "rewards/format_reward": 0.3958333432674408, | |
| "step": 217 | |
| }, | |
| { | |
| "completion_length": 2379.7709045410156, | |
| "epoch": 0.24914285714285714, | |
| "grad_norm": 0.19728457927703857, | |
| "kl": 0.00604248046875, | |
| "learning_rate": 7.243820139034464e-07, | |
| "loss": 0.0002, | |
| "reward": 0.19949759542942047, | |
| "reward_std": 0.25249316543340683, | |
| "rewards/cosine_scaled_reward": 0.07985289907082915, | |
| "rewards/format_reward": 0.5208333432674408, | |
| "step": 218 | |
| }, | |
| { | |
| "completion_length": 2062.8750762939453, | |
| "epoch": 0.2502857142857143, | |
| "grad_norm": 0.24292118847370148, | |
| "kl": 0.0070343017578125, | |
| "learning_rate": 7.214816693576234e-07, | |
| "loss": 0.0003, | |
| "reward": 0.17538858950138092, | |
| "reward_std": 0.29175762832164764, | |
| "rewards/cosine_scaled_reward": 0.010988058522343636, | |
| "rewards/format_reward": 0.6250000204890966, | |
| "step": 219 | |
| }, | |
| { | |
| "completion_length": 2316.2083435058594, | |
| "epoch": 0.25142857142857145, | |
| "grad_norm": 0.23384706676006317, | |
| "kl": 0.0063934326171875, | |
| "learning_rate": 7.185729670371604e-07, | |
| "loss": 0.0003, | |
| "reward": 0.1457906775176525, | |
| "reward_std": 0.3087628111243248, | |
| "rewards/cosine_scaled_reward": 0.0019795289263129234, | |
| "rewards/format_reward": 0.5625000055879354, | |
| "step": 220 | |
| }, | |
| { | |
| "completion_length": 2980.979217529297, | |
| "epoch": 0.25257142857142856, | |
| "grad_norm": 0.16741648316383362, | |
| "kl": 0.00563812255859375, | |
| "learning_rate": 7.156560487081051e-07, | |
| "loss": 0.0002, | |
| "reward": 0.06579325348138809, | |
| "reward_std": 0.22489114850759506, | |
| "rewards/cosine_scaled_reward": -0.004787076264619827, | |
| "rewards/format_reward": 0.2500000111758709, | |
| "step": 221 | |
| }, | |
| { | |
| "completion_length": 3021.2291870117188, | |
| "epoch": 0.2537142857142857, | |
| "grad_norm": 0.16808848083019257, | |
| "kl": 0.0062408447265625, | |
| "learning_rate": 7.127310565369415e-07, | |
| "loss": 0.0002, | |
| "reward": -0.12883399985730648, | |
| "reward_std": 0.161459781229496, | |
| "rewards/cosine_scaled_reward": -0.2722409665584564, | |
| "rewards/format_reward": 0.31250000558793545, | |
| "step": 222 | |
| }, | |
| { | |
| "completion_length": 2279.0000610351562, | |
| "epoch": 0.25485714285714284, | |
| "grad_norm": 0.22214451432228088, | |
| "kl": 0.006351470947265625, | |
| "learning_rate": 7.097981330836616e-07, | |
| "loss": 0.0003, | |
| "reward": 0.1543485686997883, | |
| "reward_std": 0.3306116834282875, | |
| "rewards/cosine_scaled_reward": -0.030070938169956207, | |
| "rewards/format_reward": 0.6875000298023224, | |
| "step": 223 | |
| }, | |
| { | |
| "completion_length": 2914.041748046875, | |
| "epoch": 0.256, | |
| "grad_norm": 0.19999858736991882, | |
| "kl": 0.006381988525390625, | |
| "learning_rate": 7.068574212948169e-07, | |
| "loss": 0.0003, | |
| "reward": 0.12693990394473076, | |
| "reward_std": 0.26520421355962753, | |
| "rewards/cosine_scaled_reward": 0.027581635862588882, | |
| "rewards/format_reward": 0.39583334140479565, | |
| "step": 224 | |
| }, | |
| { | |
| "completion_length": 1888.7083435058594, | |
| "epoch": 0.2571428571428571, | |
| "grad_norm": 0.2063429355621338, | |
| "kl": 0.004634857177734375, | |
| "learning_rate": 7.039090644965509e-07, | |
| "loss": 0.0002, | |
| "reward": 0.2952297478914261, | |
| "reward_std": 0.3628021404147148, | |
| "rewards/cosine_scaled_reward": 0.12950634956359863, | |
| "rewards/format_reward": 0.7500000149011612, | |
| "step": 225 | |
| }, | |
| { | |
| "completion_length": 2112.5833740234375, | |
| "epoch": 0.2582857142857143, | |
| "grad_norm": 0.25479966402053833, | |
| "kl": 0.0068817138671875, | |
| "learning_rate": 7.009532063876148e-07, | |
| "loss": 0.0003, | |
| "reward": 0.016783216036856174, | |
| "reward_std": 0.17216098122298717, | |
| "rewards/cosine_scaled_reward": -0.17684979364275932, | |
| "rewards/format_reward": 0.6041666716337204, | |
| "step": 226 | |
| }, | |
| { | |
| "completion_length": 2380.8958740234375, | |
| "epoch": 0.25942857142857145, | |
| "grad_norm": 0.26156362891197205, | |
| "kl": 0.009124755859375, | |
| "learning_rate": 6.979899910323624e-07, | |
| "loss": 0.0004, | |
| "reward": 0.15587822534143925, | |
| "reward_std": 0.19110089540481567, | |
| "rewards/cosine_scaled_reward": 0.041788652539253235, | |
| "rewards/format_reward": 0.47916667722165585, | |
| "step": 227 | |
| }, | |
| { | |
| "completion_length": 2745.1458740234375, | |
| "epoch": 0.26057142857142856, | |
| "grad_norm": 0.21084943413734436, | |
| "kl": 0.008335113525390625, | |
| "learning_rate": 6.950195628537299e-07, | |
| "loss": 0.0003, | |
| "reward": -0.011233052238821983, | |
| "reward_std": 0.2842288613319397, | |
| "rewards/cosine_scaled_reward": -0.13671706430613995, | |
| "rewards/format_reward": 0.375, | |
| "step": 228 | |
| }, | |
| { | |
| "completion_length": 2820.2084045410156, | |
| "epoch": 0.26171428571428573, | |
| "grad_norm": 0.2139682024717331, | |
| "kl": 0.00830078125, | |
| "learning_rate": 6.920420666261961e-07, | |
| "loss": 0.0003, | |
| "reward": -0.028370358049869537, | |
| "reward_std": 0.22242224216461182, | |
| "rewards/cosine_scaled_reward": -0.14231275767087936, | |
| "rewards/format_reward": 0.31250000558793545, | |
| "step": 229 | |
| }, | |
| { | |
| "completion_length": 2517.6458435058594, | |
| "epoch": 0.26285714285714284, | |
| "grad_norm": 0.24873493611812592, | |
| "kl": 0.005519866943359375, | |
| "learning_rate": 6.890576474687263e-07, | |
| "loss": 0.0002, | |
| "reward": 0.07593339681625366, | |
| "reward_std": 0.2139127440750599, | |
| "rewards/cosine_scaled_reward": -0.06714182905852795, | |
| "rewards/format_reward": 0.4791666716337204, | |
| "step": 230 | |
| }, | |
| { | |
| "completion_length": 3043.354248046875, | |
| "epoch": 0.264, | |
| "grad_norm": 0.21457058191299438, | |
| "kl": 0.005893707275390625, | |
| "learning_rate": 6.860664508377001e-07, | |
| "loss": 0.0002, | |
| "reward": 0.051117491617333144, | |
| "reward_std": 0.3278818652033806, | |
| "rewards/cosine_scaled_reward": -0.0638924278318882, | |
| "rewards/format_reward": 0.3750000074505806, | |
| "step": 231 | |
| }, | |
| { | |
| "completion_length": 2739.3959350585938, | |
| "epoch": 0.2651428571428571, | |
| "grad_norm": 0.17329609394073486, | |
| "kl": 0.00769805908203125, | |
| "learning_rate": 6.83068622519821e-07, | |
| "loss": 0.0003, | |
| "reward": 0.021644635125994682, | |
| "reward_std": 0.2958267964422703, | |
| "rewards/cosine_scaled_reward": -0.11065343767404556, | |
| "rewards/format_reward": 0.4166666679084301, | |
| "step": 232 | |
| }, | |
| { | |
| "completion_length": 2905.9584350585938, | |
| "epoch": 0.2662857142857143, | |
| "grad_norm": 0.19478535652160645, | |
| "kl": 0.00627899169921875, | |
| "learning_rate": 6.800643086250121e-07, | |
| "loss": 0.0003, | |
| "reward": 0.04925770778208971, | |
| "reward_std": 0.27384280413389206, | |
| "rewards/cosine_scaled_reward": -0.07401277404278517, | |
| "rewards/format_reward": 0.4166666828095913, | |
| "step": 233 | |
| }, | |
| { | |
| "completion_length": 2620.4791870117188, | |
| "epoch": 0.2674285714285714, | |
| "grad_norm": 0.2395622432231903, | |
| "kl": 0.005405426025390625, | |
| "learning_rate": 6.770536555792944e-07, | |
| "loss": 0.0002, | |
| "reward": 0.14990946277976036, | |
| "reward_std": 0.24759047105908394, | |
| "rewards/cosine_scaled_reward": 0.008792880922555923, | |
| "rewards/format_reward": 0.5625000149011612, | |
| "step": 234 | |
| }, | |
| { | |
| "completion_length": 2266.6458435058594, | |
| "epoch": 0.26857142857142857, | |
| "grad_norm": 0.22145743668079376, | |
| "kl": 0.00689697265625, | |
| "learning_rate": 6.740368101176495e-07, | |
| "loss": 0.0003, | |
| "reward": 0.1657513678073883, | |
| "reward_std": 0.20570813119411469, | |
| "rewards/cosine_scaled_reward": 0.020839732140302658, | |
| "rewards/format_reward": 0.5625000074505806, | |
| "step": 235 | |
| }, | |
| { | |
| "completion_length": 3213.0834350585938, | |
| "epoch": 0.26971428571428574, | |
| "grad_norm": 0.16639220714569092, | |
| "kl": 0.00777435302734375, | |
| "learning_rate": 6.710139192768694e-07, | |
| "loss": 0.0003, | |
| "reward": -0.11140311323106289, | |
| "reward_std": 0.2293899804353714, | |
| "rewards/cosine_scaled_reward": -0.21164199709892273, | |
| "rewards/format_reward": 0.20833333395421505, | |
| "step": 236 | |
| }, | |
| { | |
| "completion_length": 2067.041717529297, | |
| "epoch": 0.27085714285714285, | |
| "grad_norm": 0.22358138859272003, | |
| "kl": 0.00661468505859375, | |
| "learning_rate": 6.679851303883891e-07, | |
| "loss": 0.0003, | |
| "reward": 0.17607398540712893, | |
| "reward_std": 0.2522607706487179, | |
| "rewards/cosine_scaled_reward": 0.010511822998523712, | |
| "rewards/format_reward": 0.6458333432674408, | |
| "step": 237 | |
| }, | |
| { | |
| "completion_length": 2907.1250610351562, | |
| "epoch": 0.272, | |
| "grad_norm": 0.1881432682275772, | |
| "kl": 0.0087432861328125, | |
| "learning_rate": 6.649505910711058e-07, | |
| "loss": 0.0004, | |
| "reward": 0.05860769626451656, | |
| "reward_std": 0.27075210958719254, | |
| "rewards/cosine_scaled_reward": -0.06318384176120162, | |
| "rewards/format_reward": 0.39583334513008595, | |
| "step": 238 | |
| }, | |
| { | |
| "completion_length": 2931.2709350585938, | |
| "epoch": 0.27314285714285713, | |
| "grad_norm": 0.1813727766275406, | |
| "kl": 0.00714111328125, | |
| "learning_rate": 6.619104492241847e-07, | |
| "loss": 0.0003, | |
| "reward": -0.06584363710135221, | |
| "reward_std": 0.169918742030859, | |
| "rewards/cosine_scaled_reward": -0.18867192044854164, | |
| "rewards/format_reward": 0.3125000111758709, | |
| "step": 239 | |
| }, | |
| { | |
| "completion_length": 2581.291717529297, | |
| "epoch": 0.2742857142857143, | |
| "grad_norm": 0.19227683544158936, | |
| "kl": 0.00748443603515625, | |
| "learning_rate": 6.588648530198504e-07, | |
| "loss": 0.0003, | |
| "reward": 0.35164642333984375, | |
| "reward_std": 0.21003012545406818, | |
| "rewards/cosine_scaled_reward": 0.28084333799779415, | |
| "rewards/format_reward": 0.5208333414047956, | |
| "step": 240 | |
| }, | |
| { | |
| "completion_length": 2414.0000610351562, | |
| "epoch": 0.2754285714285714, | |
| "grad_norm": 0.2242266684770584, | |
| "kl": 0.0083160400390625, | |
| "learning_rate": 6.558139508961654e-07, | |
| "loss": 0.0003, | |
| "reward": 0.0253978269174695, | |
| "reward_std": 0.23190999776124954, | |
| "rewards/cosine_scaled_reward": -0.15006050281226635, | |
| "rewards/format_reward": 0.541666679084301, | |
| "step": 241 | |
| }, | |
| { | |
| "completion_length": 2462.979217529297, | |
| "epoch": 0.2765714285714286, | |
| "grad_norm": 0.2282198965549469, | |
| "kl": 0.00994873046875, | |
| "learning_rate": 6.527578915497951e-07, | |
| "loss": 0.0004, | |
| "reward": 0.10085699986666441, | |
| "reward_std": 0.30299263074994087, | |
| "rewards/cosine_scaled_reward": -0.061745526269078255, | |
| "rewards/format_reward": 0.5625000149011612, | |
| "step": 242 | |
| }, | |
| { | |
| "completion_length": 2193.8750610351562, | |
| "epoch": 0.2777142857142857, | |
| "grad_norm": 0.2123919129371643, | |
| "kl": 0.00720977783203125, | |
| "learning_rate": 6.496968239287603e-07, | |
| "loss": 0.0003, | |
| "reward": 0.05669710412621498, | |
| "reward_std": 0.2819325216114521, | |
| "rewards/cosine_scaled_reward": -0.12275011092424393, | |
| "rewards/format_reward": 0.5833333432674408, | |
| "step": 243 | |
| }, | |
| { | |
| "completion_length": 2698.104248046875, | |
| "epoch": 0.27885714285714286, | |
| "grad_norm": 0.16919925808906555, | |
| "kl": 0.005615234375, | |
| "learning_rate": 6.466308972251785e-07, | |
| "loss": 0.0002, | |
| "reward": 0.12239578366279602, | |
| "reward_std": 0.3395903930068016, | |
| "rewards/cosine_scaled_reward": -0.010242638178169727, | |
| "rewards/format_reward": 0.5, | |
| "step": 244 | |
| }, | |
| { | |
| "completion_length": 2342.8958740234375, | |
| "epoch": 0.28, | |
| "grad_norm": 0.22528305649757385, | |
| "kl": 0.00743865966796875, | |
| "learning_rate": 6.435602608679916e-07, | |
| "loss": 0.0003, | |
| "reward": 0.0906514166854322, | |
| "reward_std": 0.2340186908841133, | |
| "rewards/cosine_scaled_reward": -0.07136806473135948, | |
| "rewards/format_reward": 0.5625000111758709, | |
| "step": 245 | |
| }, | |
| { | |
| "completion_length": 2473.6250610351562, | |
| "epoch": 0.28114285714285714, | |
| "grad_norm": 0.2235722541809082, | |
| "kl": 0.00926971435546875, | |
| "learning_rate": 6.404850645156841e-07, | |
| "loss": 0.0004, | |
| "reward": 0.08842453034594655, | |
| "reward_std": 0.2991355210542679, | |
| "rewards/cosine_scaled_reward": -0.0602152943611145, | |
| "rewards/format_reward": 0.520833358168602, | |
| "step": 246 | |
| }, | |
| { | |
| "completion_length": 1907.9792175292969, | |
| "epoch": 0.2822857142857143, | |
| "grad_norm": 0.24586226046085358, | |
| "kl": 0.00750732421875, | |
| "learning_rate": 6.374054580489873e-07, | |
| "loss": 0.0003, | |
| "reward": 0.39153139293193817, | |
| "reward_std": 0.43457452207803726, | |
| "rewards/cosine_scaled_reward": 0.23923185467720032, | |
| "rewards/format_reward": 0.7708333507180214, | |
| "step": 247 | |
| }, | |
| { | |
| "completion_length": 2732.729278564453, | |
| "epoch": 0.2834285714285714, | |
| "grad_norm": 0.35658103227615356, | |
| "kl": 0.0088958740234375, | |
| "learning_rate": 6.343215915635761e-07, | |
| "loss": 0.0004, | |
| "reward": 0.06365766655653715, | |
| "reward_std": 0.344327948987484, | |
| "rewards/cosine_scaled_reward": -0.06142286770045757, | |
| "rewards/format_reward": 0.41666667349636555, | |
| "step": 248 | |
| }, | |
| { | |
| "completion_length": 3000.3125610351562, | |
| "epoch": 0.2845714285714286, | |
| "grad_norm": 0.17208239436149597, | |
| "kl": 0.006988525390625, | |
| "learning_rate": 6.31233615362752e-07, | |
| "loss": 0.0003, | |
| "reward": 0.09354665782302618, | |
| "reward_std": 0.30788997933268547, | |
| "rewards/cosine_scaled_reward": 0.0037511512637138367, | |
| "rewards/format_reward": 0.33333334513008595, | |
| "step": 249 | |
| }, | |
| { | |
| "completion_length": 2316.8333740234375, | |
| "epoch": 0.2857142857142857, | |
| "grad_norm": 0.20622476935386658, | |
| "kl": 0.00716400146484375, | |
| "learning_rate": 6.281416799501187e-07, | |
| "loss": 0.0003, | |
| "reward": 0.21881220489740372, | |
| "reward_std": 0.42582786083221436, | |
| "rewards/cosine_scaled_reward": 0.07656998513266444, | |
| "rewards/format_reward": 0.604166692122817, | |
| "step": 250 | |
| }, | |
| { | |
| "completion_length": 3048.8750610351562, | |
| "epoch": 0.28685714285714287, | |
| "grad_norm": 0.18378515541553497, | |
| "kl": 0.008056640625, | |
| "learning_rate": 6.25045936022246e-07, | |
| "loss": 0.0003, | |
| "reward": 0.03178184013813734, | |
| "reward_std": 0.3299577981233597, | |
| "rewards/cosine_scaled_reward": -0.07074778061360121, | |
| "rewards/format_reward": 0.3333333395421505, | |
| "step": 251 | |
| }, | |
| { | |
| "completion_length": 2436.041717529297, | |
| "epoch": 0.288, | |
| "grad_norm": 0.2138858288526535, | |
| "kl": 0.007293701171875, | |
| "learning_rate": 6.219465344613258e-07, | |
| "loss": 0.0003, | |
| "reward": 0.16548656299710274, | |
| "reward_std": 0.24069152027368546, | |
| "rewards/cosine_scaled_reward": 0.003951352089643478, | |
| "rewards/format_reward": 0.6041666865348816, | |
| "step": 252 | |
| }, | |
| { | |
| "completion_length": 2785.041748046875, | |
| "epoch": 0.28914285714285715, | |
| "grad_norm": 0.36376261711120605, | |
| "kl": 0.01300048828125, | |
| "learning_rate": 6.188436263278172e-07, | |
| "loss": 0.0005, | |
| "reward": -0.04214626643806696, | |
| "reward_std": 0.27475107461214066, | |
| "rewards/cosine_scaled_reward": -0.18151451274752617, | |
| "rewards/format_reward": 0.3750000074505806, | |
| "step": 253 | |
| }, | |
| { | |
| "completion_length": 1763.5000610351562, | |
| "epoch": 0.29028571428571426, | |
| "grad_norm": 0.24696381390094757, | |
| "kl": 0.00769805908203125, | |
| "learning_rate": 6.157373628530852e-07, | |
| "loss": 0.0003, | |
| "reward": 0.36900394409894943, | |
| "reward_std": 0.284480482339859, | |
| "rewards/cosine_scaled_reward": 0.23359035700559616, | |
| "rewards/format_reward": 0.7291666939854622, | |
| "step": 254 | |
| }, | |
| { | |
| "completion_length": 2458.1459045410156, | |
| "epoch": 0.2914285714285714, | |
| "grad_norm": 0.21684472262859344, | |
| "kl": 0.00672149658203125, | |
| "learning_rate": 6.126278954320294e-07, | |
| "loss": 0.0003, | |
| "reward": 0.18040073942393064, | |
| "reward_std": 0.19080796465277672, | |
| "rewards/cosine_scaled_reward": 0.06468204781413078, | |
| "rewards/format_reward": 0.5000000298023224, | |
| "step": 255 | |
| }, | |
| { | |
| "completion_length": 1878.9375305175781, | |
| "epoch": 0.2925714285714286, | |
| "grad_norm": 0.20044869184494019, | |
| "kl": 0.00609588623046875, | |
| "learning_rate": 6.095153756157051e-07, | |
| "loss": 0.0002, | |
| "reward": 0.34711842332035303, | |
| "reward_std": 0.1859663426876068, | |
| "rewards/cosine_scaled_reward": 0.14590951800346375, | |
| "rewards/format_reward": 0.9166666716337204, | |
| "step": 256 | |
| }, | |
| { | |
| "completion_length": 2303.916778564453, | |
| "epoch": 0.2937142857142857, | |
| "grad_norm": 0.2247123122215271, | |
| "kl": 0.00698089599609375, | |
| "learning_rate": 6.06399955103937e-07, | |
| "loss": 0.0003, | |
| "reward": -0.01890793489292264, | |
| "reward_std": 0.17758683115243912, | |
| "rewards/cosine_scaled_reward": -0.22834208607673645, | |
| "rewards/format_reward": 0.604166679084301, | |
| "step": 257 | |
| }, | |
| { | |
| "completion_length": 2148.8958740234375, | |
| "epoch": 0.2948571428571429, | |
| "grad_norm": 0.20504705607891083, | |
| "kl": 0.0059661865234375, | |
| "learning_rate": 6.032817857379256e-07, | |
| "loss": 0.0002, | |
| "reward": 0.2120364010334015, | |
| "reward_std": 0.29609010368585587, | |
| "rewards/cosine_scaled_reward": 0.03294338658452034, | |
| "rewards/format_reward": 0.7291666716337204, | |
| "step": 258 | |
| }, | |
| { | |
| "completion_length": 1559.3542175292969, | |
| "epoch": 0.296, | |
| "grad_norm": 0.2891344428062439, | |
| "kl": 0.00691986083984375, | |
| "learning_rate": 6.001610194928464e-07, | |
| "loss": 0.0003, | |
| "reward": 0.09913755720481277, | |
| "reward_std": 0.2612408846616745, | |
| "rewards/cosine_scaled_reward": -0.12094379169866443, | |
| "rewards/format_reward": 0.7500000149011612, | |
| "step": 259 | |
| }, | |
| { | |
| "completion_length": 2554.3959350585938, | |
| "epoch": 0.29714285714285715, | |
| "grad_norm": 0.20883695781230927, | |
| "kl": 0.00881195068359375, | |
| "learning_rate": 5.97037808470444e-07, | |
| "loss": 0.0004, | |
| "reward": 0.01960124378092587, | |
| "reward_std": 0.26494671776890755, | |
| "rewards/cosine_scaled_reward": -0.14566705003380775, | |
| "rewards/format_reward": 0.5, | |
| "step": 260 | |
| }, | |
| { | |
| "completion_length": 1605.2083740234375, | |
| "epoch": 0.29828571428571427, | |
| "grad_norm": 0.23563401401042938, | |
| "kl": 0.0057525634765625, | |
| "learning_rate": 5.939123048916173e-07, | |
| "loss": 0.0002, | |
| "reward": 0.05563092790544033, | |
| "reward_std": 0.22800205647945404, | |
| "rewards/cosine_scaled_reward": -0.18945505563169718, | |
| "rewards/format_reward": 0.7708333432674408, | |
| "step": 261 | |
| }, | |
| { | |
| "completion_length": 2781.1458740234375, | |
| "epoch": 0.29942857142857143, | |
| "grad_norm": 0.1892234981060028, | |
| "kl": 0.01024627685546875, | |
| "learning_rate": 5.907846610890011e-07, | |
| "loss": 0.0004, | |
| "reward": -0.13614844344556332, | |
| "reward_std": 0.15351427905261517, | |
| "rewards/cosine_scaled_reward": -0.29414647817611694, | |
| "rewards/format_reward": 0.3541666679084301, | |
| "step": 262 | |
| }, | |
| { | |
| "completion_length": 1101.1667175292969, | |
| "epoch": 0.30057142857142854, | |
| "grad_norm": 0.29607102274894714, | |
| "kl": 0.0075836181640625, | |
| "learning_rate": 5.87655029499542e-07, | |
| "loss": 0.0003, | |
| "reward": 0.22132008988410234, | |
| "reward_std": 0.20786399394273758, | |
| "rewards/cosine_scaled_reward": -0.02792854979634285, | |
| "rewards/format_reward": 0.9375000149011612, | |
| "step": 263 | |
| }, | |
| { | |
| "completion_length": 2106.3333740234375, | |
| "epoch": 0.3017142857142857, | |
| "grad_norm": 0.22846685349941254, | |
| "kl": 0.00617218017578125, | |
| "learning_rate": 5.845235626570683e-07, | |
| "loss": 0.0002, | |
| "reward": 0.17461178824305534, | |
| "reward_std": 0.27014725655317307, | |
| "rewards/cosine_scaled_reward": -0.04605349525809288, | |
| "rewards/format_reward": 0.791666679084301, | |
| "step": 264 | |
| }, | |
| { | |
| "completion_length": 2958.354248046875, | |
| "epoch": 0.3028571428571429, | |
| "grad_norm": 0.18784429132938385, | |
| "kl": 0.006103515625, | |
| "learning_rate": 5.813904131848564e-07, | |
| "loss": 0.0002, | |
| "reward": 0.024637079797685146, | |
| "reward_std": 0.23631998896598816, | |
| "rewards/cosine_scaled_reward": -0.08748724311590195, | |
| "rewards/format_reward": 0.3541666679084301, | |
| "step": 265 | |
| }, | |
| { | |
| "completion_length": 2931.3750610351562, | |
| "epoch": 0.304, | |
| "grad_norm": 0.1994905024766922, | |
| "kl": 0.0108184814453125, | |
| "learning_rate": 5.78255733788191e-07, | |
| "loss": 0.0004, | |
| "reward": -0.10243229102343321, | |
| "reward_std": 0.1996695175766945, | |
| "rewards/cosine_scaled_reward": -0.24480824172496796, | |
| "rewards/format_reward": 0.33333334513008595, | |
| "step": 266 | |
| }, | |
| { | |
| "completion_length": 2326.854248046875, | |
| "epoch": 0.30514285714285716, | |
| "grad_norm": 0.20324274897575378, | |
| "kl": 0.0056915283203125, | |
| "learning_rate": 5.751196772469237e-07, | |
| "loss": 0.0002, | |
| "reward": 0.18746926821768284, | |
| "reward_std": 0.24190597608685493, | |
| "rewards/cosine_scaled_reward": 0.026176687330007553, | |
| "rewards/format_reward": 0.6458333432674408, | |
| "step": 267 | |
| }, | |
| { | |
| "completion_length": 2314.5625610351562, | |
| "epoch": 0.3062857142857143, | |
| "grad_norm": 0.19881677627563477, | |
| "kl": 0.0066680908203125, | |
| "learning_rate": 5.71982396408026e-07, | |
| "loss": 0.0003, | |
| "reward": 0.10610839445143938, | |
| "reward_std": 0.23754945397377014, | |
| "rewards/cosine_scaled_reward": -0.07855316065251827, | |
| "rewards/format_reward": 0.6458333432674408, | |
| "step": 268 | |
| }, | |
| { | |
| "completion_length": 1667.7708740234375, | |
| "epoch": 0.30742857142857144, | |
| "grad_norm": 0.2681999206542969, | |
| "kl": 0.00818634033203125, | |
| "learning_rate": 5.688440441781398e-07, | |
| "loss": 0.0003, | |
| "reward": 0.13219817238859832, | |
| "reward_std": 0.17853804863989353, | |
| "rewards/cosine_scaled_reward": -0.08864733949303627, | |
| "rewards/format_reward": 0.7708333432674408, | |
| "step": 269 | |
| }, | |
| { | |
| "completion_length": 1063.7291717529297, | |
| "epoch": 0.30857142857142855, | |
| "grad_norm": 0.2545178532600403, | |
| "kl": 0.0053863525390625, | |
| "learning_rate": 5.657047735161255e-07, | |
| "loss": 0.0002, | |
| "reward": 0.23251443728804588, | |
| "reward_std": 0.2816649228334427, | |
| "rewards/cosine_scaled_reward": -0.02202252857387066, | |
| "rewards/format_reward": 0.9583333432674408, | |
| "step": 270 | |
| }, | |
| { | |
| "completion_length": 2213.3958740234375, | |
| "epoch": 0.3097142857142857, | |
| "grad_norm": 0.186069056391716, | |
| "kl": 0.00629425048828125, | |
| "learning_rate": 5.625647374256061e-07, | |
| "loss": 0.0003, | |
| "reward": 0.2779443562030792, | |
| "reward_std": 0.22687088139355183, | |
| "rewards/cosine_scaled_reward": 0.13549616560339928, | |
| "rewards/format_reward": 0.6666666772216558, | |
| "step": 271 | |
| }, | |
| { | |
| "completion_length": 3206.4375610351562, | |
| "epoch": 0.31085714285714283, | |
| "grad_norm": 0.17066144943237305, | |
| "kl": 0.0094451904296875, | |
| "learning_rate": 5.594240889475106e-07, | |
| "loss": 0.0004, | |
| "reward": -0.0621743812225759, | |
| "reward_std": 0.26253468357026577, | |
| "rewards/cosine_scaled_reward": -0.15979204699397087, | |
| "rewards/format_reward": 0.2500000037252903, | |
| "step": 272 | |
| }, | |
| { | |
| "completion_length": 2643.3125610351562, | |
| "epoch": 0.312, | |
| "grad_norm": 0.2409026324748993, | |
| "kl": 0.00916290283203125, | |
| "learning_rate": 5.562829811526154e-07, | |
| "loss": 0.0004, | |
| "reward": -0.04340513329952955, | |
| "reward_std": 0.22447111830115318, | |
| "rewards/cosine_scaled_reward": -0.21603597328066826, | |
| "rewards/format_reward": 0.479166679084301, | |
| "step": 273 | |
| }, | |
| { | |
| "completion_length": 2134.4583740234375, | |
| "epoch": 0.31314285714285717, | |
| "grad_norm": 0.22451116144657135, | |
| "kl": 0.0064697265625, | |
| "learning_rate": 5.531415671340826e-07, | |
| "loss": 0.0003, | |
| "reward": 0.15485219238325953, | |
| "reward_std": 0.27546945214271545, | |
| "rewards/cosine_scaled_reward": -0.03998738154768944, | |
| "rewards/format_reward": 0.7083333432674408, | |
| "step": 274 | |
| }, | |
| { | |
| "completion_length": 2609.3333740234375, | |
| "epoch": 0.3142857142857143, | |
| "grad_norm": 0.2192443460226059, | |
| "kl": 0.0079498291015625, | |
| "learning_rate": 5.5e-07, | |
| "loss": 0.0003, | |
| "reward": 0.07758530229330063, | |
| "reward_std": 0.25556471943855286, | |
| "rewards/cosine_scaled_reward": -0.09019988495856524, | |
| "rewards/format_reward": 0.5625000149011612, | |
| "step": 275 | |
| }, | |
| { | |
| "completion_length": 1755.5625915527344, | |
| "epoch": 0.31542857142857145, | |
| "grad_norm": 0.2460157871246338, | |
| "kl": 0.0076751708984375, | |
| "learning_rate": 5.468584328659172e-07, | |
| "loss": 0.0003, | |
| "reward": 0.211357356980443, | |
| "reward_std": 0.2688931114971638, | |
| "rewards/cosine_scaled_reward": 0.01429477147758007, | |
| "rewards/format_reward": 0.7708333432674408, | |
| "step": 276 | |
| }, | |
| { | |
| "completion_length": 1832.6250305175781, | |
| "epoch": 0.31657142857142856, | |
| "grad_norm": 0.2850809395313263, | |
| "kl": 0.00719451904296875, | |
| "learning_rate": 5.437170188473847e-07, | |
| "loss": 0.0003, | |
| "reward": 0.1353502692654729, | |
| "reward_std": 0.28056155517697334, | |
| "rewards/cosine_scaled_reward": -0.0737249106168747, | |
| "rewards/format_reward": 0.7291666865348816, | |
| "step": 277 | |
| }, | |
| { | |
| "completion_length": 2390.979217529297, | |
| "epoch": 0.3177142857142857, | |
| "grad_norm": 0.2092747986316681, | |
| "kl": 0.0069122314453125, | |
| "learning_rate": 5.405759110524894e-07, | |
| "loss": 0.0003, | |
| "reward": 0.24532984290271997, | |
| "reward_std": 0.2878542058169842, | |
| "rewards/cosine_scaled_reward": 0.0971972830593586, | |
| "rewards/format_reward": 0.6458333544433117, | |
| "step": 278 | |
| }, | |
| { | |
| "completion_length": 2098.854248046875, | |
| "epoch": 0.31885714285714284, | |
| "grad_norm": 0.235019713640213, | |
| "kl": 0.00823211669921875, | |
| "learning_rate": 5.37435262574394e-07, | |
| "loss": 0.0003, | |
| "reward": 0.10473778727464378, | |
| "reward_std": 0.26891491189599037, | |
| "rewards/cosine_scaled_reward": -0.07466491125524044, | |
| "rewards/format_reward": 0.6250000074505806, | |
| "step": 279 | |
| }, | |
| { | |
| "completion_length": 1949.9584350585938, | |
| "epoch": 0.32, | |
| "grad_norm": 0.16544176638126373, | |
| "kl": 0.005886077880859375, | |
| "learning_rate": 5.342952264838747e-07, | |
| "loss": 0.0002, | |
| "reward": 0.25803492963314056, | |
| "reward_std": 0.22913503646850586, | |
| "rewards/cosine_scaled_reward": 0.03127211146056652, | |
| "rewards/format_reward": 0.8958333432674408, | |
| "step": 280 | |
| }, | |
| { | |
| "completion_length": 1686.1875305175781, | |
| "epoch": 0.3211428571428571, | |
| "grad_norm": 0.231206014752388, | |
| "kl": 0.007114410400390625, | |
| "learning_rate": 5.311559558218603e-07, | |
| "loss": 0.0003, | |
| "reward": 0.29051591642200947, | |
| "reward_std": 0.301223948597908, | |
| "rewards/cosine_scaled_reward": 0.09080295264720917, | |
| "rewards/format_reward": 0.8541666716337204, | |
| "step": 281 | |
| }, | |
| { | |
| "completion_length": 2663.0000915527344, | |
| "epoch": 0.3222857142857143, | |
| "grad_norm": 0.16389000415802002, | |
| "kl": 0.0096282958984375, | |
| "learning_rate": 5.28017603591974e-07, | |
| "loss": 0.0004, | |
| "reward": 0.10802310952567495, | |
| "reward_std": 0.37506720423698425, | |
| "rewards/cosine_scaled_reward": -0.0454124566167593, | |
| "rewards/format_reward": 0.5625000149011612, | |
| "step": 282 | |
| }, | |
| { | |
| "completion_length": 1539.3959045410156, | |
| "epoch": 0.32342857142857145, | |
| "grad_norm": 0.2511042654514313, | |
| "kl": 0.0092620849609375, | |
| "learning_rate": 5.248803227530763e-07, | |
| "loss": 0.0004, | |
| "reward": 0.06443386664614081, | |
| "reward_std": 0.17517686262726784, | |
| "rewards/cosine_scaled_reward": -0.1600881894119084, | |
| "rewards/format_reward": 0.7291666716337204, | |
| "step": 283 | |
| }, | |
| { | |
| "completion_length": 1875.6458435058594, | |
| "epoch": 0.32457142857142857, | |
| "grad_norm": 0.2905344069004059, | |
| "kl": 0.00885009765625, | |
| "learning_rate": 5.21744266211809e-07, | |
| "loss": 0.0004, | |
| "reward": 0.15612581057939678, | |
| "reward_std": 0.26716162264347076, | |
| "rewards/cosine_scaled_reward": -0.07781252264976501, | |
| "rewards/format_reward": 0.8333333432674408, | |
| "step": 284 | |
| }, | |
| { | |
| "completion_length": 2329.666717529297, | |
| "epoch": 0.32571428571428573, | |
| "grad_norm": 0.1955825239419937, | |
| "kl": 0.00830841064453125, | |
| "learning_rate": 5.186095868151436e-07, | |
| "loss": 0.0003, | |
| "reward": -0.014332971069961786, | |
| "reward_std": 0.2384210340678692, | |
| "rewards/cosine_scaled_reward": -0.22018385492265224, | |
| "rewards/format_reward": 0.6041666716337204, | |
| "step": 285 | |
| }, | |
| { | |
| "completion_length": 1986.8126220703125, | |
| "epoch": 0.32685714285714285, | |
| "grad_norm": 0.2045590579509735, | |
| "kl": 0.00740814208984375, | |
| "learning_rate": 5.154764373429315e-07, | |
| "loss": 0.0003, | |
| "reward": 0.2143438160419464, | |
| "reward_std": 0.33449699729681015, | |
| "rewards/cosine_scaled_reward": 0.02582629583775997, | |
| "rewards/format_reward": 0.7500000149011612, | |
| "step": 286 | |
| }, | |
| { | |
| "completion_length": 2453.8333740234375, | |
| "epoch": 0.328, | |
| "grad_norm": 0.1958751082420349, | |
| "kl": 0.0113067626953125, | |
| "learning_rate": 5.123449705004581e-07, | |
| "loss": 0.0005, | |
| "reward": -0.0014798734337091446, | |
| "reward_std": 0.20798438042402267, | |
| "rewards/cosine_scaled_reward": -0.1620903555303812, | |
| "rewards/format_reward": 0.4791666865348816, | |
| "step": 287 | |
| }, | |
| { | |
| "completion_length": 1948.229248046875, | |
| "epoch": 0.3291428571428571, | |
| "grad_norm": 0.27305203676223755, | |
| "kl": 0.011627197265625, | |
| "learning_rate": 5.09215338910999e-07, | |
| "loss": 0.0005, | |
| "reward": 0.1596199981868267, | |
| "reward_std": 0.16709502786397934, | |
| "rewards/cosine_scaled_reward": -0.02836191887035966, | |
| "rewards/format_reward": 0.708333358168602, | |
| "step": 288 | |
| }, | |
| { | |
| "completion_length": 1896.1250305175781, | |
| "epoch": 0.3302857142857143, | |
| "grad_norm": 0.2106921523809433, | |
| "kl": 0.00792694091796875, | |
| "learning_rate": 5.060876951083828e-07, | |
| "loss": 0.0003, | |
| "reward": 0.1784962136298418, | |
| "reward_std": 0.19687382131814957, | |
| "rewards/cosine_scaled_reward": -0.04321554955095053, | |
| "rewards/format_reward": 0.8125000149011612, | |
| "step": 289 | |
| }, | |
| { | |
| "completion_length": 1664.1458587646484, | |
| "epoch": 0.3314285714285714, | |
| "grad_norm": 0.21843799948692322, | |
| "kl": 0.0081329345703125, | |
| "learning_rate": 5.02962191529556e-07, | |
| "loss": 0.0003, | |
| "reward": 0.5732791125774384, | |
| "reward_std": 0.17374157533049583, | |
| "rewards/cosine_scaled_reward": 0.41034475713968277, | |
| "rewards/format_reward": 0.9791666716337204, | |
| "step": 290 | |
| }, | |
| { | |
| "completion_length": 1389.666732788086, | |
| "epoch": 0.3325714285714286, | |
| "grad_norm": 0.31224629282951355, | |
| "kl": 0.00982666015625, | |
| "learning_rate": 4.998389805071536e-07, | |
| "loss": 0.0004, | |
| "reward": 0.2626770590431988, | |
| "reward_std": 0.22773029655218124, | |
| "rewards/cosine_scaled_reward": 0.07469475641846657, | |
| "rewards/format_reward": 0.7916666716337204, | |
| "step": 291 | |
| }, | |
| { | |
| "completion_length": 1609.6458740234375, | |
| "epoch": 0.33371428571428574, | |
| "grad_norm": 0.27222633361816406, | |
| "kl": 0.009613037109375, | |
| "learning_rate": 4.967182142620745e-07, | |
| "loss": 0.0004, | |
| "reward": 0.10958927869796753, | |
| "reward_std": 0.2859853506088257, | |
| "rewards/cosine_scaled_reward": -0.1261476818472147, | |
| "rewards/format_reward": 0.7916666716337204, | |
| "step": 292 | |
| }, | |
| { | |
| "completion_length": 2273.5834350585938, | |
| "epoch": 0.33485714285714285, | |
| "grad_norm": 0.2254333794116974, | |
| "kl": 0.01031494140625, | |
| "learning_rate": 4.93600044896063e-07, | |
| "loss": 0.0004, | |
| "reward": 0.10152808134444058, | |
| "reward_std": 0.29285991936922073, | |
| "rewards/cosine_scaled_reward": -0.08520433306694031, | |
| "rewards/format_reward": 0.6458333432674408, | |
| "step": 293 | |
| }, | |
| { | |
| "completion_length": 1862.7917175292969, | |
| "epoch": 0.336, | |
| "grad_norm": 0.186554417014122, | |
| "kl": 0.00743865966796875, | |
| "learning_rate": 4.904846243842949e-07, | |
| "loss": 0.0003, | |
| "reward": 0.2801675386726856, | |
| "reward_std": 0.3269800990819931, | |
| "rewards/cosine_scaled_reward": 0.0798236517002806, | |
| "rewards/format_reward": 0.8333333432674408, | |
| "step": 294 | |
| }, | |
| { | |
| "completion_length": 2408.729248046875, | |
| "epoch": 0.33714285714285713, | |
| "grad_norm": 0.1892344355583191, | |
| "kl": 0.0077056884765625, | |
| "learning_rate": 4.873721045679706e-07, | |
| "loss": 0.0003, | |
| "reward": 0.1762567274272442, | |
| "reward_std": 0.2879105731844902, | |
| "rewards/cosine_scaled_reward": 0.032621614169329405, | |
| "rewards/format_reward": 0.5833333432674408, | |
| "step": 295 | |
| }, | |
| { | |
| "completion_length": 1477.7708892822266, | |
| "epoch": 0.3382857142857143, | |
| "grad_norm": 0.3810408413410187, | |
| "kl": 0.007415771484375, | |
| "learning_rate": 4.842626371469149e-07, | |
| "loss": 0.0003, | |
| "reward": 0.30762497521936893, | |
| "reward_std": 0.30592041462659836, | |
| "rewards/cosine_scaled_reward": 0.09398248294746736, | |
| "rewards/format_reward": 0.8958333432674408, | |
| "step": 296 | |
| }, | |
| { | |
| "completion_length": 2005.8333740234375, | |
| "epoch": 0.3394285714285714, | |
| "grad_norm": 0.23766450583934784, | |
| "kl": 0.0084686279296875, | |
| "learning_rate": 4.811563736721829e-07, | |
| "loss": 0.0003, | |
| "reward": 0.10483757872134447, | |
| "reward_std": 0.23863688111305237, | |
| "rewards/cosine_scaled_reward": -0.0952923595905304, | |
| "rewards/format_reward": 0.6875000149011612, | |
| "step": 297 | |
| }, | |
| { | |
| "completion_length": 1948.2709045410156, | |
| "epoch": 0.3405714285714286, | |
| "grad_norm": 0.22391702234745026, | |
| "kl": 0.0073699951171875, | |
| "learning_rate": 4.780534655386743e-07, | |
| "loss": 0.0003, | |
| "reward": 0.1197497546672821, | |
| "reward_std": 0.2647099569439888, | |
| "rewards/cosine_scaled_reward": -0.10869306535460055, | |
| "rewards/format_reward": 0.7916666716337204, | |
| "step": 298 | |
| }, | |
| { | |
| "completion_length": 1697.2084045410156, | |
| "epoch": 0.3417142857142857, | |
| "grad_norm": 0.2413056641817093, | |
| "kl": 0.00830078125, | |
| "learning_rate": 4.749540639777539e-07, | |
| "loss": 0.0003, | |
| "reward": 0.18542808666825294, | |
| "reward_std": 0.27115339785814285, | |
| "rewards/cosine_scaled_reward": -0.01068168506026268, | |
| "rewards/format_reward": 0.7500000149011612, | |
| "step": 299 | |
| }, | |
| { | |
| "completion_length": 1569.3125457763672, | |
| "epoch": 0.34285714285714286, | |
| "grad_norm": 0.21255984902381897, | |
| "kl": 0.00722503662109375, | |
| "learning_rate": 4.7185832004988133e-07, | |
| "loss": 0.0003, | |
| "reward": 0.3450528532266617, | |
| "reward_std": 0.2651615794748068, | |
| "rewards/cosine_scaled_reward": 0.15506408301735064, | |
| "rewards/format_reward": 0.8541666716337204, | |
| "step": 300 | |
| }, | |
| { | |
| "completion_length": 2331.666717529297, | |
| "epoch": 0.344, | |
| "grad_norm": 0.27719616889953613, | |
| "kl": 0.0101776123046875, | |
| "learning_rate": 4.68766384637248e-07, | |
| "loss": 0.0004, | |
| "reward": 0.16701876651495695, | |
| "reward_std": 0.26342786848545074, | |
| "rewards/cosine_scaled_reward": 0.03490189649164677, | |
| "rewards/format_reward": 0.5416666902601719, | |
| "step": 301 | |
| }, | |
| { | |
| "completion_length": 2015.4584045410156, | |
| "epoch": 0.34514285714285714, | |
| "grad_norm": 0.27819642424583435, | |
| "kl": 0.00850677490234375, | |
| "learning_rate": 4.656784084364238e-07, | |
| "loss": 0.0003, | |
| "reward": 0.09659364819526672, | |
| "reward_std": 0.12537105567753315, | |
| "rewards/cosine_scaled_reward": -0.09112071990966797, | |
| "rewards/format_reward": 0.6458333432674408, | |
| "step": 302 | |
| }, | |
| { | |
| "completion_length": 1485.9166870117188, | |
| "epoch": 0.3462857142857143, | |
| "grad_norm": 0.26828280091285706, | |
| "kl": 0.00751495361328125, | |
| "learning_rate": 4.6259454195101267e-07, | |
| "loss": 0.0003, | |
| "reward": 0.25270497193560004, | |
| "reward_std": 0.2928587533533573, | |
| "rewards/cosine_scaled_reward": 0.027904170332476497, | |
| "rewards/format_reward": 0.8750000149011612, | |
| "step": 303 | |
| }, | |
| { | |
| "completion_length": 1409.7917175292969, | |
| "epoch": 0.3474285714285714, | |
| "grad_norm": 0.2540818452835083, | |
| "kl": 0.006877899169921875, | |
| "learning_rate": 4.59514935484316e-07, | |
| "loss": 0.0003, | |
| "reward": 0.5040274113416672, | |
| "reward_std": 0.35688790678977966, | |
| "rewards/cosine_scaled_reward": 0.34280968457460403, | |
| "rewards/format_reward": 0.8958333432674408, | |
| "step": 304 | |
| }, | |
| { | |
| "completion_length": 2382.0208740234375, | |
| "epoch": 0.3485714285714286, | |
| "grad_norm": 0.19963636994361877, | |
| "kl": 0.009735107421875, | |
| "learning_rate": 4.5643973913200837e-07, | |
| "loss": 0.0004, | |
| "reward": 0.11577347759157419, | |
| "reward_std": 0.30082143656909466, | |
| "rewards/cosine_scaled_reward": -0.03267804719507694, | |
| "rewards/format_reward": 0.5416666865348816, | |
| "step": 305 | |
| }, | |
| { | |
| "completion_length": 1835.479248046875, | |
| "epoch": 0.3497142857142857, | |
| "grad_norm": 0.28124022483825684, | |
| "kl": 0.009246826171875, | |
| "learning_rate": 4.5336910277482155e-07, | |
| "loss": 0.0004, | |
| "reward": 0.24207957834005356, | |
| "reward_std": 0.2189928311854601, | |
| "rewards/cosine_scaled_reward": 0.0470082089304924, | |
| "rewards/format_reward": 0.7916666865348816, | |
| "step": 306 | |
| }, | |
| { | |
| "completion_length": 1760.8334197998047, | |
| "epoch": 0.35085714285714287, | |
| "grad_norm": 0.22857794165611267, | |
| "kl": 0.00923919677734375, | |
| "learning_rate": 4.503031760712397e-07, | |
| "loss": 0.0004, | |
| "reward": 0.4835302531719208, | |
| "reward_std": 0.3119688630104065, | |
| "rewards/cosine_scaled_reward": 0.32135529443621635, | |
| "rewards/format_reward": 0.895833358168602, | |
| "step": 307 | |
| }, | |
| { | |
| "completion_length": 1406.291732788086, | |
| "epoch": 0.352, | |
| "grad_norm": 0.28863319754600525, | |
| "kl": 0.00827789306640625, | |
| "learning_rate": 4.4724210845020494e-07, | |
| "loss": 0.0003, | |
| "reward": 0.2202434539794922, | |
| "reward_std": 0.25928834080696106, | |
| "rewards/cosine_scaled_reward": 0.013711273670196533, | |
| "rewards/format_reward": 0.8125000074505806, | |
| "step": 308 | |
| }, | |
| { | |
| "completion_length": 1427.7708587646484, | |
| "epoch": 0.35314285714285715, | |
| "grad_norm": 0.2582438886165619, | |
| "kl": 0.00743865966796875, | |
| "learning_rate": 4.441860491038345e-07, | |
| "loss": 0.0003, | |
| "reward": 0.38336939364671707, | |
| "reward_std": 0.34063655138015747, | |
| "rewards/cosine_scaled_reward": 0.17991949617862701, | |
| "rewards/format_reward": 0.9375000149011612, | |
| "step": 309 | |
| }, | |
| { | |
| "completion_length": 1809.1250610351562, | |
| "epoch": 0.35428571428571426, | |
| "grad_norm": 0.2695036828517914, | |
| "kl": 0.0096435546875, | |
| "learning_rate": 4.4113514698014953e-07, | |
| "loss": 0.0004, | |
| "reward": 0.21986294770613313, | |
| "reward_std": 0.18356493674218655, | |
| "rewards/cosine_scaled_reward": 0.044077259954065084, | |
| "rewards/format_reward": 0.7083333432674408, | |
| "step": 310 | |
| }, | |
| { | |
| "completion_length": 1884.6459197998047, | |
| "epoch": 0.3554285714285714, | |
| "grad_norm": 0.29340508580207825, | |
| "kl": 0.0126800537109375, | |
| "learning_rate": 4.3808955077581546e-07, | |
| "loss": 0.0005, | |
| "reward": 0.1723873894661665, | |
| "reward_std": 0.268467765301466, | |
| "rewards/cosine_scaled_reward": -0.022497177124023438, | |
| "rewards/format_reward": 0.7291666865348816, | |
| "step": 311 | |
| }, | |
| { | |
| "completion_length": 1841.6666717529297, | |
| "epoch": 0.3565714285714286, | |
| "grad_norm": 0.24835428595542908, | |
| "kl": 0.007354736328125, | |
| "learning_rate": 4.350494089288943e-07, | |
| "loss": 0.0003, | |
| "reward": 0.16523393616080284, | |
| "reward_std": 0.24913058057427406, | |
| "rewards/cosine_scaled_reward": -0.044235333101823926, | |
| "rewards/format_reward": 0.7708333432674408, | |
| "step": 312 | |
| }, | |
| { | |
| "completion_length": 1764.6042175292969, | |
| "epoch": 0.3577142857142857, | |
| "grad_norm": 0.2532818615436554, | |
| "kl": 0.00913238525390625, | |
| "learning_rate": 4.3201486961161093e-07, | |
| "loss": 0.0004, | |
| "reward": 0.2583326920866966, | |
| "reward_std": 0.26906657963991165, | |
| "rewards/cosine_scaled_reward": 0.05645761638879776, | |
| "rewards/format_reward": 0.8125000149011612, | |
| "step": 313 | |
| }, | |
| { | |
| "completion_length": 1522.0833435058594, | |
| "epoch": 0.3588571428571429, | |
| "grad_norm": 0.23159317672252655, | |
| "kl": 0.0091400146484375, | |
| "learning_rate": 4.2898608072313045e-07, | |
| "loss": 0.0004, | |
| "reward": 0.23400770500302315, | |
| "reward_std": 0.28054384142160416, | |
| "rewards/cosine_scaled_reward": 0.020135470665991306, | |
| "rewards/format_reward": 0.8333333432674408, | |
| "step": 314 | |
| }, | |
| { | |
| "completion_length": 1458.8958587646484, | |
| "epoch": 0.36, | |
| "grad_norm": 0.3017834722995758, | |
| "kl": 0.0092010498046875, | |
| "learning_rate": 4.2596318988235037e-07, | |
| "loss": 0.0004, | |
| "reward": 0.1311683924868703, | |
| "reward_std": 0.282433345913887, | |
| "rewards/cosine_scaled_reward": -0.08330600336194038, | |
| "rewards/format_reward": 0.7500000149011612, | |
| "step": 315 | |
| }, | |
| { | |
| "completion_length": 2081.9584045410156, | |
| "epoch": 0.36114285714285715, | |
| "grad_norm": 0.22785058617591858, | |
| "kl": 0.0100555419921875, | |
| "learning_rate": 4.2294634442070553e-07, | |
| "loss": 0.0004, | |
| "reward": 0.02211561892181635, | |
| "reward_std": 0.21739701926708221, | |
| "rewards/cosine_scaled_reward": -0.1856558918952942, | |
| "rewards/format_reward": 0.645833358168602, | |
| "step": 316 | |
| }, | |
| { | |
| "completion_length": 2084.8958435058594, | |
| "epoch": 0.36228571428571427, | |
| "grad_norm": 0.23225082457065582, | |
| "kl": 0.0111846923828125, | |
| "learning_rate": 4.1993569137498776e-07, | |
| "loss": 0.0004, | |
| "reward": 0.24845389276742935, | |
| "reward_std": 0.3764058109372854, | |
| "rewards/cosine_scaled_reward": 0.08984526246786118, | |
| "rewards/format_reward": 0.6875000149011612, | |
| "step": 317 | |
| }, | |
| { | |
| "completion_length": 1409.4584045410156, | |
| "epoch": 0.36342857142857143, | |
| "grad_norm": 0.29590070247650146, | |
| "kl": 0.0090789794921875, | |
| "learning_rate": 4.1693137748017915e-07, | |
| "loss": 0.0004, | |
| "reward": 0.24361606128513813, | |
| "reward_std": 0.24939386546611786, | |
| "rewards/cosine_scaled_reward": 0.04053226858377457, | |
| "rewards/format_reward": 0.8125000149011612, | |
| "step": 318 | |
| }, | |
| { | |
| "completion_length": 2240.375030517578, | |
| "epoch": 0.36457142857142855, | |
| "grad_norm": 0.18136313557624817, | |
| "kl": 0.00847625732421875, | |
| "learning_rate": 4.1393354916230005e-07, | |
| "loss": 0.0003, | |
| "reward": 0.08488270919770002, | |
| "reward_std": 0.22877753525972366, | |
| "rewards/cosine_scaled_reward": -0.10751745477318764, | |
| "rewards/format_reward": 0.6458333432674408, | |
| "step": 319 | |
| }, | |
| { | |
| "completion_length": 1544.0833740234375, | |
| "epoch": 0.3657142857142857, | |
| "grad_norm": 0.20819739997386932, | |
| "kl": 0.00759124755859375, | |
| "learning_rate": 4.1094235253127374e-07, | |
| "loss": 0.0003, | |
| "reward": 0.2654244042932987, | |
| "reward_std": 0.38539183512330055, | |
| "rewards/cosine_scaled_reward": 0.039359224028885365, | |
| "rewards/format_reward": 0.8958333432674408, | |
| "step": 320 | |
| }, | |
| { | |
| "completion_length": 1928.1042175292969, | |
| "epoch": 0.3668571428571429, | |
| "grad_norm": 0.24936559796333313, | |
| "kl": 0.010467529296875, | |
| "learning_rate": 4.079579333738039e-07, | |
| "loss": 0.0004, | |
| "reward": 0.2094899509102106, | |
| "reward_std": 0.2675232719630003, | |
| "rewards/cosine_scaled_reward": 0.017967309337109327, | |
| "rewards/format_reward": 0.7500000149011612, | |
| "step": 321 | |
| }, | |
| { | |
| "completion_length": 1580.6875915527344, | |
| "epoch": 0.368, | |
| "grad_norm": 0.20408599078655243, | |
| "kl": 0.00743865966796875, | |
| "learning_rate": 4.0498043714627006e-07, | |
| "loss": 0.0003, | |
| "reward": 0.26650879299268126, | |
| "reward_std": 0.291965126991272, | |
| "rewards/cosine_scaled_reward": 0.060018595308065414, | |
| "rewards/format_reward": 0.8541666865348816, | |
| "step": 322 | |
| }, | |
| { | |
| "completion_length": 1619.5833740234375, | |
| "epoch": 0.36914285714285716, | |
| "grad_norm": 0.32702451944351196, | |
| "kl": 0.01053619384765625, | |
| "learning_rate": 4.020100089676376e-07, | |
| "loss": 0.0004, | |
| "reward": 0.2938319506647531, | |
| "reward_std": 0.23641689121723175, | |
| "rewards/cosine_scaled_reward": 0.10801427066326141, | |
| "rewards/format_reward": 0.8125000149011612, | |
| "step": 323 | |
| }, | |
| { | |
| "completion_length": 2247.1876220703125, | |
| "epoch": 0.3702857142857143, | |
| "grad_norm": 0.17617079615592957, | |
| "kl": 0.01004791259765625, | |
| "learning_rate": 3.9904679361238526e-07, | |
| "loss": 0.0004, | |
| "reward": 0.06403969042003155, | |
| "reward_std": 0.18300126865506172, | |
| "rewards/cosine_scaled_reward": -0.14668823406100273, | |
| "rewards/format_reward": 0.6875, | |
| "step": 324 | |
| }, | |
| { | |
| "completion_length": 1287.2916717529297, | |
| "epoch": 0.37142857142857144, | |
| "grad_norm": 0.2974833548069, | |
| "kl": 0.0090179443359375, | |
| "learning_rate": 3.9609093550344907e-07, | |
| "loss": 0.0004, | |
| "reward": 0.2224448136985302, | |
| "reward_std": 0.23644054681062698, | |
| "rewards/cosine_scaled_reward": -0.036079500801861286, | |
| "rewards/format_reward": 0.9583333432674408, | |
| "step": 325 | |
| }, | |
| { | |
| "completion_length": 2086.7709045410156, | |
| "epoch": 0.37257142857142855, | |
| "grad_norm": 0.2205517441034317, | |
| "kl": 0.0098724365234375, | |
| "learning_rate": 3.931425787051832e-07, | |
| "loss": 0.0004, | |
| "reward": 0.09755693469196558, | |
| "reward_std": 0.21148419380187988, | |
| "rewards/cosine_scaled_reward": -0.13864601170644164, | |
| "rewards/format_reward": 0.7916666865348816, | |
| "step": 326 | |
| }, | |
| { | |
| "completion_length": 1831.3958740234375, | |
| "epoch": 0.3737142857142857, | |
| "grad_norm": 0.21228300034999847, | |
| "kl": 0.00830078125, | |
| "learning_rate": 3.902018669163384e-07, | |
| "loss": 0.0003, | |
| "reward": 0.33440493792295456, | |
| "reward_std": 0.3173820227384567, | |
| "rewards/cosine_scaled_reward": 0.1545264758169651, | |
| "rewards/format_reward": 0.833333358168602, | |
| "step": 327 | |
| }, | |
| { | |
| "completion_length": 1920.5000610351562, | |
| "epoch": 0.37485714285714283, | |
| "grad_norm": 0.2396845668554306, | |
| "kl": 0.009307861328125, | |
| "learning_rate": 3.872689434630585e-07, | |
| "loss": 0.0004, | |
| "reward": 0.2613999950699508, | |
| "reward_std": 0.3687175065279007, | |
| "rewards/cosine_scaled_reward": 0.07327798567712307, | |
| "rewards/format_reward": 0.7916666716337204, | |
| "step": 328 | |
| }, | |
| { | |
| "completion_length": 1893.8125610351562, | |
| "epoch": 0.376, | |
| "grad_norm": 0.20392858982086182, | |
| "kl": 0.008724212646484375, | |
| "learning_rate": 3.843439512918949e-07, | |
| "loss": 0.0003, | |
| "reward": 0.19521827809512615, | |
| "reward_std": 0.17398589849472046, | |
| "rewards/cosine_scaled_reward": -0.021663442254066467, | |
| "rewards/format_reward": 0.8125000074505806, | |
| "step": 329 | |
| }, | |
| { | |
| "completion_length": 1882.9583740234375, | |
| "epoch": 0.37714285714285717, | |
| "grad_norm": 0.21649308502674103, | |
| "kl": 0.00913238525390625, | |
| "learning_rate": 3.8142703296283953e-07, | |
| "loss": 0.0004, | |
| "reward": 0.13386597856879234, | |
| "reward_std": 0.27085599303245544, | |
| "rewards/cosine_scaled_reward": -0.09174733608961105, | |
| "rewards/format_reward": 0.7916666865348816, | |
| "step": 330 | |
| }, | |
| { | |
| "completion_length": 2221.0833740234375, | |
| "epoch": 0.3782857142857143, | |
| "grad_norm": 0.19338715076446533, | |
| "kl": 0.0114288330078125, | |
| "learning_rate": 3.785183306423767e-07, | |
| "loss": 0.0005, | |
| "reward": 0.03328318754211068, | |
| "reward_std": 0.2135927379131317, | |
| "rewards/cosine_scaled_reward": -0.1791954692453146, | |
| "rewards/format_reward": 0.6666666716337204, | |
| "step": 331 | |
| }, | |
| { | |
| "completion_length": 1648.7708587646484, | |
| "epoch": 0.37942857142857145, | |
| "grad_norm": 0.2684015929698944, | |
| "kl": 0.0113983154296875, | |
| "learning_rate": 3.7561798609655373e-07, | |
| "loss": 0.0005, | |
| "reward": 0.27905246056616306, | |
| "reward_std": 0.2478124052286148, | |
| "rewards/cosine_scaled_reward": 0.08956634253263474, | |
| "rewards/format_reward": 0.791666679084301, | |
| "step": 332 | |
| }, | |
| { | |
| "completion_length": 1530.3750610351562, | |
| "epoch": 0.38057142857142856, | |
| "grad_norm": 0.2717783451080322, | |
| "kl": 0.00899505615234375, | |
| "learning_rate": 3.72726140684072e-07, | |
| "loss": 0.0004, | |
| "reward": 0.05967093911021948, | |
| "reward_std": 0.20809690654277802, | |
| "rewards/cosine_scaled_reward": -0.20973167195916176, | |
| "rewards/format_reward": 0.8541666865348816, | |
| "step": 333 | |
| }, | |
| { | |
| "completion_length": 2227.5625610351562, | |
| "epoch": 0.38171428571428573, | |
| "grad_norm": 0.23560327291488647, | |
| "kl": 0.01085662841796875, | |
| "learning_rate": 3.6984293534939737e-07, | |
| "loss": 0.0004, | |
| "reward": 0.11222782591357827, | |
| "reward_std": 0.2779008559882641, | |
| "rewards/cosine_scaled_reward": -0.09507040795870125, | |
| "rewards/format_reward": 0.708333358168602, | |
| "step": 334 | |
| }, | |
| { | |
| "completion_length": 1943.4375915527344, | |
| "epoch": 0.38285714285714284, | |
| "grad_norm": 0.24016280472278595, | |
| "kl": 0.011260986328125, | |
| "learning_rate": 3.6696851061588994e-07, | |
| "loss": 0.0005, | |
| "reward": 0.19086962472647429, | |
| "reward_std": 0.2430224046111107, | |
| "rewards/cosine_scaled_reward": -0.0018318742513656616, | |
| "rewards/format_reward": 0.7291666716337204, | |
| "step": 335 | |
| }, | |
| { | |
| "completion_length": 1407.2708740234375, | |
| "epoch": 0.384, | |
| "grad_norm": 0.28007882833480835, | |
| "kl": 0.00952911376953125, | |
| "learning_rate": 3.641030065789562e-07, | |
| "loss": 0.0004, | |
| "reward": 0.05958843324333429, | |
| "reward_std": 0.1869518756866455, | |
| "rewards/cosine_scaled_reward": -0.20403875783085823, | |
| "rewards/format_reward": 0.833333358168602, | |
| "step": 336 | |
| }, | |
| { | |
| "completion_length": 1033.5625305175781, | |
| "epoch": 0.3851428571428571, | |
| "grad_norm": 0.3786531686782837, | |
| "kl": 0.0092010498046875, | |
| "learning_rate": 3.612465628992203e-07, | |
| "loss": 0.0004, | |
| "reward": 0.20698048546910286, | |
| "reward_std": 0.29393167048692703, | |
| "rewards/cosine_scaled_reward": -0.0597881693392992, | |
| "rewards/format_reward": 0.9791666716337204, | |
| "step": 337 | |
| }, | |
| { | |
| "completion_length": 2073.812530517578, | |
| "epoch": 0.3862857142857143, | |
| "grad_norm": 0.23650962114334106, | |
| "kl": 0.0127410888671875, | |
| "learning_rate": 3.5839931879571725e-07, | |
| "loss": 0.0005, | |
| "reward": 0.12870828062295914, | |
| "reward_std": 0.20198489911854267, | |
| "rewards/cosine_scaled_reward": -0.028040427714586258, | |
| "rewards/format_reward": 0.5833333432674408, | |
| "step": 338 | |
| }, | |
| { | |
| "completion_length": 2329.0833740234375, | |
| "epoch": 0.38742857142857146, | |
| "grad_norm": 0.18778569996356964, | |
| "kl": 0.0103607177734375, | |
| "learning_rate": 3.555614130391079e-07, | |
| "loss": 0.0004, | |
| "reward": 0.18835976673290133, | |
| "reward_std": 0.3419927656650543, | |
| "rewards/cosine_scaled_reward": -0.004078354686498642, | |
| "rewards/format_reward": 0.7500000223517418, | |
| "step": 339 | |
| }, | |
| { | |
| "completion_length": 1800.1042175292969, | |
| "epoch": 0.38857142857142857, | |
| "grad_norm": 0.24106545746326447, | |
| "kl": 0.0111236572265625, | |
| "learning_rate": 3.5273298394491515e-07, | |
| "loss": 0.0004, | |
| "reward": 0.32668111473321915, | |
| "reward_std": 0.3139052540063858, | |
| "rewards/cosine_scaled_reward": 0.1498567252419889, | |
| "rewards/format_reward": 0.7916666865348816, | |
| "step": 340 | |
| }, | |
| { | |
| "completion_length": 2232.354248046875, | |
| "epoch": 0.38971428571428574, | |
| "grad_norm": 0.29757753014564514, | |
| "kl": 0.01445770263671875, | |
| "learning_rate": 3.4991416936678276e-07, | |
| "loss": 0.0006, | |
| "reward": 0.284415390342474, | |
| "reward_std": 0.431053563952446, | |
| "rewards/cosine_scaled_reward": 0.15291668381541967, | |
| "rewards/format_reward": 0.6250000298023224, | |
| "step": 341 | |
| }, | |
| { | |
| "completion_length": 1537.2500610351562, | |
| "epoch": 0.39085714285714285, | |
| "grad_norm": 0.2549525797367096, | |
| "kl": 0.009307861328125, | |
| "learning_rate": 3.471051066897562e-07, | |
| "loss": 0.0004, | |
| "reward": 0.12320299912244081, | |
| "reward_std": 0.18919622898101807, | |
| "rewards/cosine_scaled_reward": -0.12590695265680552, | |
| "rewards/format_reward": 0.8541666865348816, | |
| "step": 342 | |
| }, | |
| { | |
| "completion_length": 2050.4166870117188, | |
| "epoch": 0.392, | |
| "grad_norm": 0.233432337641716, | |
| "kl": 0.01062774658203125, | |
| "learning_rate": 3.4430593282358777e-07, | |
| "loss": 0.0004, | |
| "reward": 0.43845948576927185, | |
| "reward_std": 0.2883150465786457, | |
| "rewards/cosine_scaled_reward": 0.28409258276224136, | |
| "rewards/format_reward": 0.833333358168602, | |
| "step": 343 | |
| }, | |
| { | |
| "completion_length": 1527.2500610351562, | |
| "epoch": 0.3931428571428571, | |
| "grad_norm": 0.28435570001602173, | |
| "kl": 0.0095977783203125, | |
| "learning_rate": 3.4151678419606233e-07, | |
| "loss": 0.0004, | |
| "reward": 0.2925238283351064, | |
| "reward_std": 0.2560243997722864, | |
| "rewards/cosine_scaled_reward": 0.07367146760225296, | |
| "rewards/format_reward": 0.8958333432674408, | |
| "step": 344 | |
| }, | |
| { | |
| "completion_length": 1440.1666870117188, | |
| "epoch": 0.3942857142857143, | |
| "grad_norm": 0.23948732018470764, | |
| "kl": 0.00936126708984375, | |
| "learning_rate": 3.387377967463493e-07, | |
| "loss": 0.0004, | |
| "reward": 0.1968644427252002, | |
| "reward_std": 0.1854616329073906, | |
| "rewards/cosine_scaled_reward": -0.031224748119711876, | |
| "rewards/format_reward": 0.8541666716337204, | |
| "step": 345 | |
| }, | |
| { | |
| "completion_length": 1666.041748046875, | |
| "epoch": 0.3954285714285714, | |
| "grad_norm": 0.2333936244249344, | |
| "kl": 0.0108184814453125, | |
| "learning_rate": 3.359691059183761e-07, | |
| "loss": 0.0004, | |
| "reward": 0.14479913003742695, | |
| "reward_std": 0.2107578031718731, | |
| "rewards/cosine_scaled_reward": -0.074419766664505, | |
| "rewards/format_reward": 0.7916666716337204, | |
| "step": 346 | |
| }, | |
| { | |
| "completion_length": 1407.9167175292969, | |
| "epoch": 0.3965714285714286, | |
| "grad_norm": 0.2971218228340149, | |
| "kl": 0.00939178466796875, | |
| "learning_rate": 3.3321084665422803e-07, | |
| "loss": 0.0004, | |
| "reward": 0.38189948722720146, | |
| "reward_std": 0.325411569327116, | |
| "rewards/cosine_scaled_reward": 0.20823876559734344, | |
| "rewards/format_reward": 0.8541666865348816, | |
| "step": 347 | |
| }, | |
| { | |
| "completion_length": 1421.041732788086, | |
| "epoch": 0.3977142857142857, | |
| "grad_norm": 0.2317306399345398, | |
| "kl": 0.00820159912109375, | |
| "learning_rate": 3.3046315338757026e-07, | |
| "loss": 0.0003, | |
| "reward": 0.1995458109304309, | |
| "reward_std": 0.2409229427576065, | |
| "rewards/cosine_scaled_reward": -0.03460083529353142, | |
| "rewards/format_reward": 0.875, | |
| "step": 348 | |
| }, | |
| { | |
| "completion_length": 1281.4375305175781, | |
| "epoch": 0.39885714285714285, | |
| "grad_norm": 0.24120782315731049, | |
| "kl": 0.007274627685546875, | |
| "learning_rate": 3.2772616003709616e-07, | |
| "loss": 0.0003, | |
| "reward": 0.45311590284109116, | |
| "reward_std": 0.2731959819793701, | |
| "rewards/cosine_scaled_reward": 0.25743514811620116, | |
| "rewards/format_reward": 0.9583333432674408, | |
| "step": 349 | |
| }, | |
| { | |
| "completion_length": 1245.7291870117188, | |
| "epoch": 0.4, | |
| "grad_norm": 0.22964408993721008, | |
| "kl": 0.009246826171875, | |
| "learning_rate": 3.250000000000001e-07, | |
| "loss": 0.0004, | |
| "reward": 0.3433077558875084, | |
| "reward_std": 0.2860936261713505, | |
| "rewards/cosine_scaled_reward": 0.15224721981212497, | |
| "rewards/format_reward": 0.8750000149011612, | |
| "step": 350 | |
| }, | |
| { | |
| "completion_length": 1456.7083587646484, | |
| "epoch": 0.40114285714285713, | |
| "grad_norm": 0.28671830892562866, | |
| "kl": 0.008087158203125, | |
| "learning_rate": 3.222848061454764e-07, | |
| "loss": 0.0003, | |
| "reward": 0.11195625504478812, | |
| "reward_std": 0.2711613141000271, | |
| "rewards/cosine_scaled_reward": -0.12700296379625797, | |
| "rewards/format_reward": 0.7916666716337204, | |
| "step": 351 | |
| }, | |
| { | |
| "completion_length": 1435.0416717529297, | |
| "epoch": 0.4022857142857143, | |
| "grad_norm": 0.2130868285894394, | |
| "kl": 0.0080718994140625, | |
| "learning_rate": 3.195807108082429e-07, | |
| "loss": 0.0003, | |
| "reward": 0.17259665206074715, | |
| "reward_std": 0.2315747793763876, | |
| "rewards/cosine_scaled_reward": -0.09190645813941956, | |
| "rewards/format_reward": 0.9375000149011612, | |
| "step": 352 | |
| }, | |
| { | |
| "completion_length": 1706.9583740234375, | |
| "epoch": 0.4034285714285714, | |
| "grad_norm": 0.21334676444530487, | |
| "kl": 0.01042938232421875, | |
| "learning_rate": 3.168878457820915e-07, | |
| "loss": 0.0004, | |
| "reward": 0.18855836428701878, | |
| "reward_std": 0.26317035034298897, | |
| "rewards/cosine_scaled_reward": -0.05418684799224138, | |
| "rewards/format_reward": 0.8958333432674408, | |
| "step": 353 | |
| }, | |
| { | |
| "completion_length": 1464.6875305175781, | |
| "epoch": 0.4045714285714286, | |
| "grad_norm": 0.22004835307598114, | |
| "kl": 0.009002685546875, | |
| "learning_rate": 3.142063423134644e-07, | |
| "loss": 0.0004, | |
| "reward": -0.002447195933200419, | |
| "reward_std": 0.11259411834180355, | |
| "rewards/cosine_scaled_reward": -0.30102576315402985, | |
| "rewards/format_reward": 0.8958333432674408, | |
| "step": 354 | |
| }, | |
| { | |
| "completion_length": 1175.0208435058594, | |
| "epoch": 0.4057142857142857, | |
| "grad_norm": 0.3179510831832886, | |
| "kl": 0.0092315673828125, | |
| "learning_rate": 3.115363310950578e-07, | |
| "loss": 0.0004, | |
| "reward": 0.20959803275763988, | |
| "reward_std": 0.12544530257582664, | |
| "rewards/cosine_scaled_reward": -0.0459451749920845, | |
| "rewards/format_reward": 0.9375, | |
| "step": 355 | |
| }, | |
| { | |
| "completion_length": 1753.0625610351562, | |
| "epoch": 0.40685714285714286, | |
| "grad_norm": 0.26044961810112, | |
| "kl": 0.01105499267578125, | |
| "learning_rate": 3.0887794225945143e-07, | |
| "loss": 0.0004, | |
| "reward": 0.11193252541124821, | |
| "reward_std": 0.17821012996137142, | |
| "rewards/cosine_scaled_reward": -0.1084582656621933, | |
| "rewards/format_reward": 0.7500000149011612, | |
| "step": 356 | |
| }, | |
| { | |
| "completion_length": 1933.3750915527344, | |
| "epoch": 0.408, | |
| "grad_norm": 0.32032617926597595, | |
| "kl": 0.0153656005859375, | |
| "learning_rate": 3.062313053727671e-07, | |
| "loss": 0.0006, | |
| "reward": 0.0698192942654714, | |
| "reward_std": 0.1654464676976204, | |
| "rewards/cosine_scaled_reward": -0.14839602261781693, | |
| "rewards/format_reward": 0.7083333432674408, | |
| "step": 357 | |
| }, | |
| { | |
| "completion_length": 1849.5625, | |
| "epoch": 0.40914285714285714, | |
| "grad_norm": 0.21455638110637665, | |
| "kl": 0.01059722900390625, | |
| "learning_rate": 3.0359654942835247e-07, | |
| "loss": 0.0004, | |
| "reward": 0.23701468179933727, | |
| "reward_std": 0.28458721190690994, | |
| "rewards/cosine_scaled_reward": 0.05581399705260992, | |
| "rewards/format_reward": 0.7500000074505806, | |
| "step": 358 | |
| }, | |
| { | |
| "completion_length": 1714.3333435058594, | |
| "epoch": 0.4102857142857143, | |
| "grad_norm": 0.23594503104686737, | |
| "kl": 0.0106353759765625, | |
| "learning_rate": 3.0097380284049523e-07, | |
| "loss": 0.0004, | |
| "reward": 0.17001938103931025, | |
| "reward_std": 0.2589455880224705, | |
| "rewards/cosine_scaled_reward": -0.047670695930719376, | |
| "rewards/format_reward": 0.7916667014360428, | |
| "step": 359 | |
| }, | |
| { | |
| "completion_length": 869.5416870117188, | |
| "epoch": 0.4114285714285714, | |
| "grad_norm": 0.2539467513561249, | |
| "kl": 0.00698089599609375, | |
| "learning_rate": 2.9836319343816397e-07, | |
| "loss": 0.0003, | |
| "reward": 0.38226132094860077, | |
| "reward_std": 0.2698897160589695, | |
| "rewards/cosine_scaled_reward": 0.1609980291686952, | |
| "rewards/format_reward": 0.9791666716337204, | |
| "step": 360 | |
| }, | |
| { | |
| "completion_length": 1785.3333587646484, | |
| "epoch": 0.4125714285714286, | |
| "grad_norm": 0.23149755597114563, | |
| "kl": 0.01030731201171875, | |
| "learning_rate": 2.9576484845877793e-07, | |
| "loss": 0.0004, | |
| "reward": 0.09932729974389076, | |
| "reward_std": 0.19216609373688698, | |
| "rewards/cosine_scaled_reward": -0.08814284577965736, | |
| "rewards/format_reward": 0.6458333432674408, | |
| "step": 361 | |
| }, | |
| { | |
| "completion_length": 1765.2708435058594, | |
| "epoch": 0.4137142857142857, | |
| "grad_norm": 0.25148525834083557, | |
| "kl": 0.01117706298828125, | |
| "learning_rate": 2.931788945420058e-07, | |
| "loss": 0.0004, | |
| "reward": 0.28765634074807167, | |
| "reward_std": 0.28115685284137726, | |
| "rewards/cosine_scaled_reward": 0.10722193028777838, | |
| "rewards/format_reward": 0.7916666716337204, | |
| "step": 362 | |
| }, | |
| { | |
| "completion_length": 1581.7916870117188, | |
| "epoch": 0.41485714285714287, | |
| "grad_norm": 0.2328980267047882, | |
| "kl": 0.0112457275390625, | |
| "learning_rate": 2.9060545772359305e-07, | |
| "loss": 0.0005, | |
| "reward": 0.15995342750102282, | |
| "reward_std": 0.3252422660589218, | |
| "rewards/cosine_scaled_reward": -0.09024253115057945, | |
| "rewards/format_reward": 0.8958333432674408, | |
| "step": 363 | |
| }, | |
| { | |
| "completion_length": 1506.9375457763672, | |
| "epoch": 0.416, | |
| "grad_norm": 0.28264883160591125, | |
| "kl": 0.0095977783203125, | |
| "learning_rate": 2.8804466342921987e-07, | |
| "loss": 0.0004, | |
| "reward": 0.08763573569012806, | |
| "reward_std": 0.21415090188384056, | |
| "rewards/cosine_scaled_reward": -0.16532962024211884, | |
| "rewards/format_reward": 0.833333358168602, | |
| "step": 364 | |
| }, | |
| { | |
| "completion_length": 2367.4166870117188, | |
| "epoch": 0.41714285714285715, | |
| "grad_norm": 0.19307875633239746, | |
| "kl": 0.014556884765625, | |
| "learning_rate": 2.854966364683872e-07, | |
| "loss": 0.0006, | |
| "reward": 0.20070525258779526, | |
| "reward_std": 0.26848191022872925, | |
| "rewards/cosine_scaled_reward": 0.022052939981222153, | |
| "rewards/format_reward": 0.7083333488553762, | |
| "step": 365 | |
| }, | |
| { | |
| "completion_length": 1509.9584045410156, | |
| "epoch": 0.41828571428571426, | |
| "grad_norm": 0.2306293249130249, | |
| "kl": 0.010223388671875, | |
| "learning_rate": 2.829615010283344e-07, | |
| "loss": 0.0004, | |
| "reward": 0.25371203664690256, | |
| "reward_std": 0.2659039720892906, | |
| "rewards/cosine_scaled_reward": 0.02641349472105503, | |
| "rewards/format_reward": 0.8958333432674408, | |
| "step": 366 | |
| }, | |
| { | |
| "completion_length": 1335.9167022705078, | |
| "epoch": 0.41942857142857143, | |
| "grad_norm": 0.24042809009552002, | |
| "kl": 0.00992584228515625, | |
| "learning_rate": 2.8043938066798645e-07, | |
| "loss": 0.0004, | |
| "reward": 0.3599996566772461, | |
| "reward_std": 0.19493130780756474, | |
| "rewards/cosine_scaled_reward": 0.13548098504543304, | |
| "rewards/format_reward": 0.9791666716337204, | |
| "step": 367 | |
| }, | |
| { | |
| "completion_length": 1371.1666870117188, | |
| "epoch": 0.4205714285714286, | |
| "grad_norm": 0.22017300128936768, | |
| "kl": 0.00868988037109375, | |
| "learning_rate": 2.7793039831193133e-07, | |
| "loss": 0.0003, | |
| "reward": 0.2111358353868127, | |
| "reward_std": 0.31247158348560333, | |
| "rewards/cosine_scaled_reward": -0.023564637638628483, | |
| "rewards/format_reward": 0.875, | |
| "step": 368 | |
| }, | |
| { | |
| "completion_length": 1303.375015258789, | |
| "epoch": 0.4217142857142857, | |
| "grad_norm": 0.27260446548461914, | |
| "kl": 0.009395599365234375, | |
| "learning_rate": 2.7543467624442956e-07, | |
| "loss": 0.0004, | |
| "reward": 0.39653632789850235, | |
| "reward_std": 0.1950348112732172, | |
| "rewards/cosine_scaled_reward": 0.21199024934321642, | |
| "rewards/format_reward": 0.8958333432674408, | |
| "step": 369 | |
| }, | |
| { | |
| "completion_length": 1707.4584045410156, | |
| "epoch": 0.4228571428571429, | |
| "grad_norm": 0.2327386736869812, | |
| "kl": 0.012054443359375, | |
| "learning_rate": 2.729523361034538e-07, | |
| "loss": 0.0005, | |
| "reward": 0.2546226102858782, | |
| "reward_std": 0.27051765471696854, | |
| "rewards/cosine_scaled_reward": 0.015558794140815735, | |
| "rewards/format_reward": 0.9375000149011612, | |
| "step": 370 | |
| }, | |
| { | |
| "completion_length": 1338.8125, | |
| "epoch": 0.424, | |
| "grad_norm": 0.2991894781589508, | |
| "kl": 0.0108642578125, | |
| "learning_rate": 2.7048349887476037e-07, | |
| "loss": 0.0004, | |
| "reward": 0.2581311799585819, | |
| "reward_std": 0.3103417530655861, | |
| "rewards/cosine_scaled_reward": 0.03514588810503483, | |
| "rewards/format_reward": 0.8958333432674408, | |
| "step": 371 | |
| }, | |
| { | |
| "completion_length": 1596.3958587646484, | |
| "epoch": 0.42514285714285716, | |
| "grad_norm": 0.20261621475219727, | |
| "kl": 0.00960540771484375, | |
| "learning_rate": 2.6802828488599294e-07, | |
| "loss": 0.0004, | |
| "reward": 0.23256312776356936, | |
| "reward_std": 0.182934682816267, | |
| "rewards/cosine_scaled_reward": 0.03540460020303726, | |
| "rewards/format_reward": 0.7916666865348816, | |
| "step": 372 | |
| }, | |
| { | |
| "completion_length": 1823.7917175292969, | |
| "epoch": 0.42628571428571427, | |
| "grad_norm": 0.2511698305606842, | |
| "kl": 0.01041412353515625, | |
| "learning_rate": 2.655868138008171e-07, | |
| "loss": 0.0004, | |
| "reward": 0.13830948527902365, | |
| "reward_std": 0.16817106679081917, | |
| "rewards/cosine_scaled_reward": -0.08003572002053261, | |
| "rewards/format_reward": 0.7708333432674408, | |
| "step": 373 | |
| }, | |
| { | |
| "completion_length": 1590.8750305175781, | |
| "epoch": 0.42742857142857144, | |
| "grad_norm": 0.24217580258846283, | |
| "kl": 0.0106048583984375, | |
| "learning_rate": 2.631592046130896e-07, | |
| "loss": 0.0004, | |
| "reward": 0.21486383816227317, | |
| "reward_std": 0.2895674481987953, | |
| "rewards/cosine_scaled_reward": -0.010343782603740692, | |
| "rewards/format_reward": 0.8541666865348816, | |
| "step": 374 | |
| }, | |
| { | |
| "completion_length": 1353.1458435058594, | |
| "epoch": 0.42857142857142855, | |
| "grad_norm": 0.2372012883424759, | |
| "kl": 0.01015472412109375, | |
| "learning_rate": 2.6074557564105724e-07, | |
| "loss": 0.0004, | |
| "reward": 0.3424642861355096, | |
| "reward_std": 0.25616780295968056, | |
| "rewards/cosine_scaled_reward": 0.1377451792359352, | |
| "rewards/format_reward": 0.8958333432674408, | |
| "step": 375 | |
| }, | |
| { | |
| "completion_length": 1862.2500305175781, | |
| "epoch": 0.4297142857142857, | |
| "grad_norm": 0.2163545936346054, | |
| "kl": 0.01139068603515625, | |
| "learning_rate": 2.583460445215911e-07, | |
| "loss": 0.0005, | |
| "reward": 0.29940345184877515, | |
| "reward_std": 0.2624346353113651, | |
| "rewards/cosine_scaled_reward": 0.14119264110922813, | |
| "rewards/format_reward": 0.7500000298023224, | |
| "step": 376 | |
| }, | |
| { | |
| "completion_length": 1338.4375305175781, | |
| "epoch": 0.4308571428571429, | |
| "grad_norm": 0.25990864634513855, | |
| "kl": 0.0098114013671875, | |
| "learning_rate": 2.5596072820445254e-07, | |
| "loss": 0.0004, | |
| "reward": 0.37805385142564774, | |
| "reward_std": 0.33201449923217297, | |
| "rewards/cosine_scaled_reward": 0.16646008286625147, | |
| "rewards/format_reward": 0.9375000149011612, | |
| "step": 377 | |
| }, | |
| { | |
| "completion_length": 1609.9583740234375, | |
| "epoch": 0.432, | |
| "grad_norm": 0.33538058400154114, | |
| "kl": 0.0113525390625, | |
| "learning_rate": 2.5358974294659373e-07, | |
| "loss": 0.0005, | |
| "reward": 0.07824116386473179, | |
| "reward_std": 0.23690377175807953, | |
| "rewards/cosine_scaled_reward": -0.17136050947010517, | |
| "rewards/format_reward": 0.8125000149011612, | |
| "step": 378 | |
| }, | |
| { | |
| "completion_length": 1537.2916870117188, | |
| "epoch": 0.43314285714285716, | |
| "grad_norm": 0.20783498883247375, | |
| "kl": 0.00994873046875, | |
| "learning_rate": 2.512332043064913e-07, | |
| "loss": 0.0004, | |
| "reward": 0.2669680919498205, | |
| "reward_std": 0.3232973664999008, | |
| "rewards/cosine_scaled_reward": 0.06161441095173359, | |
| "rewards/format_reward": 0.8333333432674408, | |
| "step": 379 | |
| }, | |
| { | |
| "completion_length": 1752.5000457763672, | |
| "epoch": 0.4342857142857143, | |
| "grad_norm": 0.3683948218822479, | |
| "kl": 0.014739990234375, | |
| "learning_rate": 2.488912271385139e-07, | |
| "loss": 0.0006, | |
| "reward": 0.08518872899003327, | |
| "reward_std": 0.26048473827540874, | |
| "rewards/cosine_scaled_reward": -0.12987486645579338, | |
| "rewards/format_reward": 0.708333358168602, | |
| "step": 380 | |
| }, | |
| { | |
| "completion_length": 1513.437515258789, | |
| "epoch": 0.43542857142857144, | |
| "grad_norm": 0.25296124815940857, | |
| "kl": 0.011993408203125, | |
| "learning_rate": 2.465639255873246e-07, | |
| "loss": 0.0005, | |
| "reward": 0.29362980648875237, | |
| "reward_std": 0.2426050491631031, | |
| "rewards/cosine_scaled_reward": 0.06906034797430038, | |
| "rewards/format_reward": 0.9166666716337204, | |
| "step": 381 | |
| }, | |
| { | |
| "completion_length": 1786.2917022705078, | |
| "epoch": 0.43657142857142855, | |
| "grad_norm": 0.23917359113693237, | |
| "kl": 0.0137481689453125, | |
| "learning_rate": 2.4425141308231765e-07, | |
| "loss": 0.0006, | |
| "reward": 0.14519746601581573, | |
| "reward_std": 0.25404126197099686, | |
| "rewards/cosine_scaled_reward": -0.06414372939616442, | |
| "rewards/format_reward": 0.7500000149011612, | |
| "step": 382 | |
| }, | |
| { | |
| "completion_length": 2034.9583740234375, | |
| "epoch": 0.4377142857142857, | |
| "grad_norm": 0.23685619235038757, | |
| "kl": 0.012542724609375, | |
| "learning_rate": 2.4195380233209006e-07, | |
| "loss": 0.0005, | |
| "reward": 0.17537298426032066, | |
| "reward_std": 0.19519681856036186, | |
| "rewards/cosine_scaled_reward": -0.002057701349258423, | |
| "rewards/format_reward": 0.6875000149011612, | |
| "step": 383 | |
| }, | |
| { | |
| "completion_length": 1755.5000305175781, | |
| "epoch": 0.43885714285714283, | |
| "grad_norm": 0.2406085878610611, | |
| "kl": 0.0112762451171875, | |
| "learning_rate": 2.3967120531894857e-07, | |
| "loss": 0.0005, | |
| "reward": 0.044221414253115654, | |
| "reward_std": 0.18394669890403748, | |
| "rewards/cosine_scaled_reward": -0.2134968489408493, | |
| "rewards/format_reward": 0.8125000149011612, | |
| "step": 384 | |
| }, | |
| { | |
| "completion_length": 1586.6459197998047, | |
| "epoch": 0.44, | |
| "grad_norm": 0.2748856544494629, | |
| "kl": 0.011138916015625, | |
| "learning_rate": 2.374037332934512e-07, | |
| "loss": 0.0004, | |
| "reward": 0.3660757765173912, | |
| "reward_std": 0.2511321157217026, | |
| "rewards/cosine_scaled_reward": 0.1509570861235261, | |
| "rewards/format_reward": 0.9583333432674408, | |
| "step": 385 | |
| }, | |
| { | |
| "completion_length": 1456.5625610351562, | |
| "epoch": 0.44114285714285717, | |
| "grad_norm": 0.22892530262470245, | |
| "kl": 0.0084381103515625, | |
| "learning_rate": 2.3515149676898552e-07, | |
| "loss": 0.0003, | |
| "reward": 0.16375851293560117, | |
| "reward_std": 0.22477589175105095, | |
| "rewards/cosine_scaled_reward": -0.07266581058502197, | |
| "rewards/format_reward": 0.8541666865348816, | |
| "step": 386 | |
| }, | |
| { | |
| "completion_length": 1412.1459045410156, | |
| "epoch": 0.4422857142857143, | |
| "grad_norm": 0.21265709400177002, | |
| "kl": 0.0097198486328125, | |
| "learning_rate": 2.3291460551638237e-07, | |
| "loss": 0.0004, | |
| "reward": 0.160978349391371, | |
| "reward_std": 0.25059961900115013, | |
| "rewards/cosine_scaled_reward": -0.10482261097058654, | |
| "rewards/format_reward": 0.9375, | |
| "step": 387 | |
| }, | |
| { | |
| "completion_length": 1464.2916870117188, | |
| "epoch": 0.44342857142857145, | |
| "grad_norm": 0.23378108441829681, | |
| "kl": 0.009490966796875, | |
| "learning_rate": 2.306931685585657e-07, | |
| "loss": 0.0004, | |
| "reward": 0.33504685014486313, | |
| "reward_std": 0.32544615119695663, | |
| "rewards/cosine_scaled_reward": 0.12704317644238472, | |
| "rewards/format_reward": 0.9166666865348816, | |
| "step": 388 | |
| }, | |
| { | |
| "completion_length": 1371.0416870117188, | |
| "epoch": 0.44457142857142856, | |
| "grad_norm": 0.3262147009372711, | |
| "kl": 0.01122283935546875, | |
| "learning_rate": 2.2848729416523859e-07, | |
| "loss": 0.0004, | |
| "reward": 0.393025110475719, | |
| "reward_std": 0.290339432656765, | |
| "rewards/cosine_scaled_reward": 0.21821050345897675, | |
| "rewards/format_reward": 0.8750000149011612, | |
| "step": 389 | |
| }, | |
| { | |
| "completion_length": 1398.8958892822266, | |
| "epoch": 0.44571428571428573, | |
| "grad_norm": 0.2878376841545105, | |
| "kl": 0.013916015625, | |
| "learning_rate": 2.2629708984760706e-07, | |
| "loss": 0.0006, | |
| "reward": 0.21481858170591295, | |
| "reward_std": 0.16455775126814842, | |
| "rewards/cosine_scaled_reward": 0.0046413615345954895, | |
| "rewards/format_reward": 0.8125000149011612, | |
| "step": 390 | |
| }, | |
| { | |
| "completion_length": 1230.7916870117188, | |
| "epoch": 0.44685714285714284, | |
| "grad_norm": 0.3208393156528473, | |
| "kl": 0.01119232177734375, | |
| "learning_rate": 2.2412266235313973e-07, | |
| "loss": 0.0004, | |
| "reward": 0.08701863046735525, | |
| "reward_std": 0.215634036809206, | |
| "rewards/cosine_scaled_reward": -0.20002751424908638, | |
| "rewards/format_reward": 0.9375000149011612, | |
| "step": 391 | |
| }, | |
| { | |
| "completion_length": 1943.3125610351562, | |
| "epoch": 0.448, | |
| "grad_norm": 0.20683653652668, | |
| "kl": 0.01094818115234375, | |
| "learning_rate": 2.2196411766036487e-07, | |
| "loss": 0.0004, | |
| "reward": 0.406577079789713, | |
| "reward_std": 0.31823114305734634, | |
| "rewards/cosine_scaled_reward": 0.25175799429416656, | |
| "rewards/format_reward": 0.8125, | |
| "step": 392 | |
| }, | |
| { | |
| "completion_length": 1398.9375457763672, | |
| "epoch": 0.4491428571428571, | |
| "grad_norm": 0.2916996479034424, | |
| "kl": 0.0109405517578125, | |
| "learning_rate": 2.1982156097370557e-07, | |
| "loss": 0.0004, | |
| "reward": 0.33830785006284714, | |
| "reward_std": 0.30660923570394516, | |
| "rewards/cosine_scaled_reward": 0.12406410835683346, | |
| "rewards/format_reward": 0.9166666865348816, | |
| "step": 393 | |
| }, | |
| { | |
| "completion_length": 1487.3334045410156, | |
| "epoch": 0.4502857142857143, | |
| "grad_norm": 0.29143252968788147, | |
| "kl": 0.01137542724609375, | |
| "learning_rate": 2.1769509671835223e-07, | |
| "loss": 0.0005, | |
| "reward": 0.06870951503515244, | |
| "reward_std": 0.18425077386200428, | |
| "rewards/cosine_scaled_reward": -0.2029971145093441, | |
| "rewards/format_reward": 0.8750000149011612, | |
| "step": 394 | |
| }, | |
| { | |
| "completion_length": 1707.5209045410156, | |
| "epoch": 0.4514285714285714, | |
| "grad_norm": 0.354472279548645, | |
| "kl": 0.0127716064453125, | |
| "learning_rate": 2.1558482853517253e-07, | |
| "loss": 0.0005, | |
| "reward": 0.16543664713390172, | |
| "reward_std": 0.2276711966842413, | |
| "rewards/cosine_scaled_reward": -0.03153465378272813, | |
| "rewards/format_reward": 0.7291666865348816, | |
| "step": 395 | |
| }, | |
| { | |
| "completion_length": 1278.7292022705078, | |
| "epoch": 0.45257142857142857, | |
| "grad_norm": 0.25319093465805054, | |
| "kl": 0.010589599609375, | |
| "learning_rate": 2.134908592756607e-07, | |
| "loss": 0.0004, | |
| "reward": 0.19444908574223518, | |
| "reward_std": 0.12517918460071087, | |
| "rewards/cosine_scaled_reward": -0.06452729552984238, | |
| "rewards/format_reward": 0.9375000149011612, | |
| "step": 396 | |
| }, | |
| { | |
| "completion_length": 1797.2500457763672, | |
| "epoch": 0.45371428571428574, | |
| "grad_norm": 0.30418673157691956, | |
| "kl": 0.017242431640625, | |
| "learning_rate": 2.1141329099692406e-07, | |
| "loss": 0.0007, | |
| "reward": 0.3547268435359001, | |
| "reward_std": 0.17775312066078186, | |
| "rewards/cosine_scaled_reward": 0.19013852439820766, | |
| "rewards/format_reward": 0.7916666716337204, | |
| "step": 397 | |
| }, | |
| { | |
| "completion_length": 1636.1875610351562, | |
| "epoch": 0.45485714285714285, | |
| "grad_norm": 0.21280616521835327, | |
| "kl": 0.0079193115234375, | |
| "learning_rate": 2.0935222495670968e-07, | |
| "loss": 0.0003, | |
| "reward": 0.32367637380957603, | |
| "reward_std": 0.2834791075438261, | |
| "rewards/cosine_scaled_reward": 0.12400428391993046, | |
| "rewards/format_reward": 0.8750000149011612, | |
| "step": 398 | |
| }, | |
| { | |
| "completion_length": 1898.1666870117188, | |
| "epoch": 0.456, | |
| "grad_norm": 0.232693612575531, | |
| "kl": 0.0127410888671875, | |
| "learning_rate": 2.0730776160846853e-07, | |
| "loss": 0.0005, | |
| "reward": 0.17925062775611877, | |
| "reward_std": 0.2557571642100811, | |
| "rewards/cosine_scaled_reward": -0.0575912781059742, | |
| "rewards/format_reward": 0.8541666865348816, | |
| "step": 399 | |
| }, | |
| { | |
| "completion_length": 1323.5208740234375, | |
| "epoch": 0.45714285714285713, | |
| "grad_norm": 0.3167197108268738, | |
| "kl": 0.0113067626953125, | |
| "learning_rate": 2.0528000059645995e-07, | |
| "loss": 0.0005, | |
| "reward": 0.22988794930279255, | |
| "reward_std": 0.1990835703909397, | |
| "rewards/cosine_scaled_reward": -0.005974147003144026, | |
| "rewards/format_reward": 0.8958333432674408, | |
| "step": 400 | |
| }, | |
| { | |
| "completion_length": 1017.5833740234375, | |
| "epoch": 0.4582857142857143, | |
| "grad_norm": 0.23701179027557373, | |
| "kl": 0.00727081298828125, | |
| "learning_rate": 2.032690407508949e-07, | |
| "loss": 0.0003, | |
| "reward": 0.2984974943101406, | |
| "reward_std": 0.20315679907798767, | |
| "rewards/cosine_scaled_reward": 0.05098311696201563, | |
| "rewards/format_reward": 0.9791666716337204, | |
| "step": 401 | |
| }, | |
| { | |
| "completion_length": 1881.5000610351562, | |
| "epoch": 0.4594285714285714, | |
| "grad_norm": 0.2918482720851898, | |
| "kl": 0.0158843994140625, | |
| "learning_rate": 2.0127498008311922e-07, | |
| "loss": 0.0006, | |
| "reward": 0.05151970311999321, | |
| "reward_std": 0.21448798850178719, | |
| "rewards/cosine_scaled_reward": -0.19786715414375067, | |
| "rewards/format_reward": 0.7916666716337204, | |
| "step": 402 | |
| }, | |
| { | |
| "completion_length": 1883.6250610351562, | |
| "epoch": 0.4605714285714286, | |
| "grad_norm": 0.18074828386306763, | |
| "kl": 0.01107025146484375, | |
| "learning_rate": 1.9929791578083655e-07, | |
| "loss": 0.0004, | |
| "reward": 0.18776276428252459, | |
| "reward_std": 0.3380716070532799, | |
| "rewards/cosine_scaled_reward": -0.027764197438955307, | |
| "rewards/format_reward": 0.8125000149011612, | |
| "step": 403 | |
| }, | |
| { | |
| "completion_length": 1436.1458740234375, | |
| "epoch": 0.4617142857142857, | |
| "grad_norm": 0.29037535190582275, | |
| "kl": 0.0111846923828125, | |
| "learning_rate": 1.9733794420337213e-07, | |
| "loss": 0.0004, | |
| "reward": 0.28416240587830544, | |
| "reward_std": 0.2827350050210953, | |
| "rewards/cosine_scaled_reward": 0.052190789952874184, | |
| "rewards/format_reward": 0.9375000149011612, | |
| "step": 404 | |
| }, | |
| { | |
| "completion_length": 1326.7084045410156, | |
| "epoch": 0.46285714285714286, | |
| "grad_norm": 0.21668171882629395, | |
| "kl": 0.0085601806640625, | |
| "learning_rate": 1.9539516087697517e-07, | |
| "loss": 0.0003, | |
| "reward": 0.34238045662641525, | |
| "reward_std": 0.2986739482730627, | |
| "rewards/cosine_scaled_reward": 0.1431681104004383, | |
| "rewards/format_reward": 0.8958333432674408, | |
| "step": 405 | |
| }, | |
| { | |
| "completion_length": 1571.479248046875, | |
| "epoch": 0.464, | |
| "grad_norm": 0.24584002792835236, | |
| "kl": 0.01373291015625, | |
| "learning_rate": 1.934696604901642e-07, | |
| "loss": 0.0005, | |
| "reward": 0.15168224461376667, | |
| "reward_std": 0.2871460150927305, | |
| "rewards/cosine_scaled_reward": -0.07575275376439095, | |
| "rewards/format_reward": 0.8125000149011612, | |
| "step": 406 | |
| }, | |
| { | |
| "completion_length": 1145.8541870117188, | |
| "epoch": 0.46514285714285714, | |
| "grad_norm": 0.38704681396484375, | |
| "kl": 0.0112457275390625, | |
| "learning_rate": 1.915615368891117e-07, | |
| "loss": 0.0004, | |
| "reward": 0.26085772132501006, | |
| "reward_std": 0.1329988930374384, | |
| "rewards/cosine_scaled_reward": 0.019920717924833298, | |
| "rewards/format_reward": 0.9375000149011612, | |
| "step": 407 | |
| }, | |
| { | |
| "completion_length": 1228.9583740234375, | |
| "epoch": 0.4662857142857143, | |
| "grad_norm": 0.28655827045440674, | |
| "kl": 0.01071929931640625, | |
| "learning_rate": 1.8967088307307e-07, | |
| "loss": 0.0004, | |
| "reward": 0.20901603996753693, | |
| "reward_std": 0.2725791148841381, | |
| "rewards/cosine_scaled_reward": -0.05561388377100229, | |
| "rewards/format_reward": 0.9583333432674408, | |
| "step": 408 | |
| }, | |
| { | |
| "completion_length": 1253.9375305175781, | |
| "epoch": 0.4674285714285714, | |
| "grad_norm": 0.23769833147525787, | |
| "kl": 0.00876617431640625, | |
| "learning_rate": 1.8779779118983867e-07, | |
| "loss": 0.0004, | |
| "reward": 0.3615655303001404, | |
| "reward_std": 0.27604348957538605, | |
| "rewards/cosine_scaled_reward": 0.15084571414627135, | |
| "rewards/format_reward": 0.9375, | |
| "step": 409 | |
| }, | |
| { | |
| "completion_length": 1575.5208435058594, | |
| "epoch": 0.4685714285714286, | |
| "grad_norm": 0.23627842962741852, | |
| "kl": 0.0123291015625, | |
| "learning_rate": 1.8594235253127372e-07, | |
| "loss": 0.0005, | |
| "reward": 0.12202017195522785, | |
| "reward_std": 0.2631267737597227, | |
| "rewards/cosine_scaled_reward": -0.13389795599505305, | |
| "rewards/format_reward": 0.8750000149011612, | |
| "step": 410 | |
| }, | |
| { | |
| "completion_length": 1910.4792175292969, | |
| "epoch": 0.4697142857142857, | |
| "grad_norm": 0.2608468234539032, | |
| "kl": 0.014801025390625, | |
| "learning_rate": 1.8410465752883758e-07, | |
| "loss": 0.0006, | |
| "reward": 0.08060921769356355, | |
| "reward_std": 0.21683987975120544, | |
| "rewards/cosine_scaled_reward": -0.14538796059787273, | |
| "rewards/format_reward": 0.7500000149011612, | |
| "step": 411 | |
| }, | |
| { | |
| "completion_length": 1346.1041717529297, | |
| "epoch": 0.47085714285714286, | |
| "grad_norm": 0.2915378212928772, | |
| "kl": 0.00960540771484375, | |
| "learning_rate": 1.822847957491922e-07, | |
| "loss": 0.0004, | |
| "reward": 0.1485822768881917, | |
| "reward_std": 0.23674238845705986, | |
| "rewards/cosine_scaled_reward": -0.1019090199843049, | |
| "rewards/format_reward": 0.8750000149011612, | |
| "step": 412 | |
| }, | |
| { | |
| "completion_length": 1397.8333740234375, | |
| "epoch": 0.472, | |
| "grad_norm": 0.2737977206707001, | |
| "kl": 0.0107574462890625, | |
| "learning_rate": 1.804828558898332e-07, | |
| "loss": 0.0004, | |
| "reward": 0.14930985309183598, | |
| "reward_std": 0.2523481696844101, | |
| "rewards/cosine_scaled_reward": -0.11631065257824957, | |
| "rewards/format_reward": 0.9166666716337204, | |
| "step": 413 | |
| }, | |
| { | |
| "completion_length": 1642.3958740234375, | |
| "epoch": 0.47314285714285714, | |
| "grad_norm": 0.20642879605293274, | |
| "kl": 0.01294708251953125, | |
| "learning_rate": 1.7869892577476722e-07, | |
| "loss": 0.0005, | |
| "reward": 0.1973862424492836, | |
| "reward_std": 0.18661364912986755, | |
| "rewards/cosine_scaled_reward": -0.008776968927122653, | |
| "rewards/format_reward": 0.7916666865348816, | |
| "step": 414 | |
| }, | |
| { | |
| "completion_length": 1843.4375305175781, | |
| "epoch": 0.4742857142857143, | |
| "grad_norm": 0.2866264581680298, | |
| "kl": 0.0138397216796875, | |
| "learning_rate": 1.7693309235023127e-07, | |
| "loss": 0.0006, | |
| "reward": 0.20638167671859264, | |
| "reward_std": 0.2785125896334648, | |
| "rewards/cosine_scaled_reward": -0.01411110907793045, | |
| "rewards/format_reward": 0.8333333432674408, | |
| "step": 415 | |
| }, | |
| { | |
| "completion_length": 1825.0625915527344, | |
| "epoch": 0.4754285714285714, | |
| "grad_norm": 0.2105662226676941, | |
| "kl": 0.0146026611328125, | |
| "learning_rate": 1.7518544168045524e-07, | |
| "loss": 0.0006, | |
| "reward": 0.1510641649365425, | |
| "reward_std": 0.33063989877700806, | |
| "rewards/cosine_scaled_reward": -0.07132141478359699, | |
| "rewards/format_reward": 0.7916666865348816, | |
| "step": 416 | |
| }, | |
| { | |
| "completion_length": 1851.4583892822266, | |
| "epoch": 0.4765714285714286, | |
| "grad_norm": 0.2664584815502167, | |
| "kl": 0.0132293701171875, | |
| "learning_rate": 1.7345605894346726e-07, | |
| "loss": 0.0005, | |
| "reward": 0.17481167800724506, | |
| "reward_std": 0.3025582581758499, | |
| "rewards/cosine_scaled_reward": -0.03878038749098778, | |
| "rewards/format_reward": 0.7916666865348816, | |
| "step": 417 | |
| }, | |
| { | |
| "completion_length": 1656.7916870117188, | |
| "epoch": 0.4777142857142857, | |
| "grad_norm": 0.33390721678733826, | |
| "kl": 0.011993408203125, | |
| "learning_rate": 1.7174502842694212e-07, | |
| "loss": 0.0005, | |
| "reward": 0.10759061248973012, | |
| "reward_std": 0.19447487592697144, | |
| "rewards/cosine_scaled_reward": -0.15984330605715513, | |
| "rewards/format_reward": 0.895833358168602, | |
| "step": 418 | |
| }, | |
| { | |
| "completion_length": 1700.5625762939453, | |
| "epoch": 0.47885714285714287, | |
| "grad_norm": 0.27000099420547485, | |
| "kl": 0.0132293701171875, | |
| "learning_rate": 1.7005243352409333e-07, | |
| "loss": 0.0005, | |
| "reward": 0.16961859352886677, | |
| "reward_std": 0.3320206254720688, | |
| "rewards/cosine_scaled_reward": -0.06891552917659283, | |
| "rewards/format_reward": 0.8541666865348816, | |
| "step": 419 | |
| }, | |
| { | |
| "completion_length": 1130.6250457763672, | |
| "epoch": 0.48, | |
| "grad_norm": 0.32490241527557373, | |
| "kl": 0.010345458984375, | |
| "learning_rate": 1.6837835672960831e-07, | |
| "loss": 0.0004, | |
| "reward": 0.16695750327198766, | |
| "reward_std": 0.22090371325612068, | |
| "rewards/cosine_scaled_reward": -0.09449976186442655, | |
| "rewards/format_reward": 0.9166666716337204, | |
| "step": 420 | |
| }, | |
| { | |
| "completion_length": 1289.3542175292969, | |
| "epoch": 0.48114285714285715, | |
| "grad_norm": 0.2906915247440338, | |
| "kl": 0.00969696044921875, | |
| "learning_rate": 1.6672287963562852e-07, | |
| "loss": 0.0004, | |
| "reward": 0.34764280915260315, | |
| "reward_std": 0.21982120722532272, | |
| "rewards/cosine_scaled_reward": 0.12819887697696686, | |
| "rewards/format_reward": 0.9375000149011612, | |
| "step": 421 | |
| }, | |
| { | |
| "completion_length": 1322.8750610351562, | |
| "epoch": 0.48228571428571426, | |
| "grad_norm": 0.21419624984264374, | |
| "kl": 0.00823974609375, | |
| "learning_rate": 1.6508608292777203e-07, | |
| "loss": 0.0003, | |
| "reward": 0.17968564480543137, | |
| "reward_std": 0.2470797523856163, | |
| "rewards/cosine_scaled_reward": -0.0866120639257133, | |
| "rewards/format_reward": 0.9583333432674408, | |
| "step": 422 | |
| }, | |
| { | |
| "completion_length": 1337.8958358764648, | |
| "epoch": 0.48342857142857143, | |
| "grad_norm": 0.3047991693019867, | |
| "kl": 0.0133056640625, | |
| "learning_rate": 1.6346804638120098e-07, | |
| "loss": 0.0005, | |
| "reward": 0.1798581276088953, | |
| "reward_std": 0.20611854176968336, | |
| "rewards/cosine_scaled_reward": -0.062100378796458244, | |
| "rewards/format_reward": 0.8750000149011612, | |
| "step": 423 | |
| }, | |
| { | |
| "completion_length": 1513.2708740234375, | |
| "epoch": 0.4845714285714286, | |
| "grad_norm": 0.2347906529903412, | |
| "kl": 0.01027679443359375, | |
| "learning_rate": 1.6186884885673413e-07, | |
| "loss": 0.0004, | |
| "reward": 0.16043103858828545, | |
| "reward_std": 0.24068767204880714, | |
| "rewards/cosine_scaled_reward": -0.09871538355946541, | |
| "rewards/format_reward": 0.9166666716337204, | |
| "step": 424 | |
| }, | |
| { | |
| "completion_length": 1886.8541870117188, | |
| "epoch": 0.4857142857142857, | |
| "grad_norm": 0.2786445915699005, | |
| "kl": 0.015716552734375, | |
| "learning_rate": 1.6028856829700258e-07, | |
| "loss": 0.0006, | |
| "reward": 0.1020525568164885, | |
| "reward_std": 0.2838766872882843, | |
| "rewards/cosine_scaled_reward": -0.14031901117414236, | |
| "rewards/format_reward": 0.8125, | |
| "step": 425 | |
| }, | |
| { | |
| "completion_length": 1222.5625305175781, | |
| "epoch": 0.4868571428571429, | |
| "grad_norm": 0.26595538854599, | |
| "kl": 0.0072479248046875, | |
| "learning_rate": 1.5872728172265146e-07, | |
| "loss": 0.0003, | |
| "reward": 0.3471098020672798, | |
| "reward_std": 0.32343775779008865, | |
| "rewards/cosine_scaled_reward": 0.12477114424109459, | |
| "rewards/format_reward": 0.9583333432674408, | |
| "step": 426 | |
| }, | |
| { | |
| "completion_length": 1624.166748046875, | |
| "epoch": 0.488, | |
| "grad_norm": 0.285559743642807, | |
| "kl": 0.01145172119140625, | |
| "learning_rate": 1.5718506522858572e-07, | |
| "loss": 0.0005, | |
| "reward": 0.3360773101449013, | |
| "reward_std": 0.2637428678572178, | |
| "rewards/cosine_scaled_reward": 0.16991135850548744, | |
| "rewards/format_reward": 0.7916666865348816, | |
| "step": 427 | |
| }, | |
| { | |
| "completion_length": 2089.6459045410156, | |
| "epoch": 0.48914285714285716, | |
| "grad_norm": 0.29123032093048096, | |
| "kl": 0.01389312744140625, | |
| "learning_rate": 1.5566199398026147e-07, | |
| "loss": 0.0006, | |
| "reward": 0.07632875768467784, | |
| "reward_std": 0.21408921107649803, | |
| "rewards/cosine_scaled_reward": -0.15921480767428875, | |
| "rewards/format_reward": 0.7708333432674408, | |
| "step": 428 | |
| }, | |
| { | |
| "completion_length": 1952.9376220703125, | |
| "epoch": 0.49028571428571427, | |
| "grad_norm": 0.2803601920604706, | |
| "kl": 0.018218994140625, | |
| "learning_rate": 1.5415814221002265e-07, | |
| "loss": 0.0007, | |
| "reward": 0.1619062926620245, | |
| "reward_std": 0.21095814555883408, | |
| "rewards/cosine_scaled_reward": -0.0001234300434589386, | |
| "rewards/format_reward": 0.6250000298023224, | |
| "step": 429 | |
| }, | |
| { | |
| "completion_length": 2041.8334350585938, | |
| "epoch": 0.49142857142857144, | |
| "grad_norm": 0.35983848571777344, | |
| "kl": 0.018157958984375, | |
| "learning_rate": 1.5267358321348285e-07, | |
| "loss": 0.0007, | |
| "reward": 0.018218230456113815, | |
| "reward_std": 0.22815853729844093, | |
| "rewards/cosine_scaled_reward": -0.2138291234150529, | |
| "rewards/format_reward": 0.708333358168602, | |
| "step": 430 | |
| }, | |
| { | |
| "completion_length": 1465.6042022705078, | |
| "epoch": 0.49257142857142855, | |
| "grad_norm": 0.38481760025024414, | |
| "kl": 0.018310546875, | |
| "learning_rate": 1.5120838934595337e-07, | |
| "loss": 0.0007, | |
| "reward": 0.25773806248616893, | |
| "reward_std": 0.24154233559966087, | |
| "rewards/cosine_scaled_reward": 0.06686290970537812, | |
| "rewards/format_reward": 0.7916666865348816, | |
| "step": 431 | |
| }, | |
| { | |
| "completion_length": 1565.5416870117188, | |
| "epoch": 0.4937142857142857, | |
| "grad_norm": 0.28354302048683167, | |
| "kl": 0.01001739501953125, | |
| "learning_rate": 1.4976263201891613e-07, | |
| "loss": 0.0004, | |
| "reward": 0.17629767209291458, | |
| "reward_std": 0.2845665439963341, | |
| "rewards/cosine_scaled_reward": -0.09245277754962444, | |
| "rewards/format_reward": 0.9583333432674408, | |
| "step": 432 | |
| }, | |
| { | |
| "completion_length": 1747.5417175292969, | |
| "epoch": 0.4948571428571429, | |
| "grad_norm": 0.28452399373054504, | |
| "kl": 0.0126953125, | |
| "learning_rate": 1.483363816965435e-07, | |
| "loss": 0.0005, | |
| "reward": 0.18188877776265144, | |
| "reward_std": 0.23814743384718895, | |
| "rewards/cosine_scaled_reward": -0.05783041566610336, | |
| "rewards/format_reward": 0.8750000149011612, | |
| "step": 433 | |
| }, | |
| { | |
| "completion_length": 1306.6875457763672, | |
| "epoch": 0.496, | |
| "grad_norm": 0.4342614412307739, | |
| "kl": 0.0144195556640625, | |
| "learning_rate": 1.469297078922642e-07, | |
| "loss": 0.0006, | |
| "reward": 0.4066113345324993, | |
| "reward_std": 0.24682630226016045, | |
| "rewards/cosine_scaled_reward": 0.19056972488760948, | |
| "rewards/format_reward": 0.9791666716337204, | |
| "step": 434 | |
| }, | |
| { | |
| "completion_length": 1489.3750305175781, | |
| "epoch": 0.49714285714285716, | |
| "grad_norm": 0.420890212059021, | |
| "kl": 0.0156402587890625, | |
| "learning_rate": 1.4554267916537495e-07, | |
| "loss": 0.0006, | |
| "reward": 0.1405728820245713, | |
| "reward_std": 0.242431428283453, | |
| "rewards/cosine_scaled_reward": -0.11428239196538925, | |
| "rewards/format_reward": 0.8750000298023224, | |
| "step": 435 | |
| }, | |
| { | |
| "completion_length": 1935.479248046875, | |
| "epoch": 0.4982857142857143, | |
| "grad_norm": 0.3561446964740753, | |
| "kl": 0.0189971923828125, | |
| "learning_rate": 1.4417536311769885e-07, | |
| "loss": 0.0008, | |
| "reward": 0.1535082533955574, | |
| "reward_std": 0.30283405259251595, | |
| "rewards/cosine_scaled_reward": -0.058026134967803955, | |
| "rewards/format_reward": 0.770833358168602, | |
| "step": 436 | |
| }, | |
| { | |
| "completion_length": 1268.5834045410156, | |
| "epoch": 0.49942857142857144, | |
| "grad_norm": 0.2628932297229767, | |
| "kl": 0.00823974609375, | |
| "learning_rate": 1.4282782639029128e-07, | |
| "loss": 0.0003, | |
| "reward": 0.26019781455397606, | |
| "reward_std": 0.22013483569025993, | |
| "rewards/cosine_scaled_reward": 0.04217514023184776, | |
| "rewards/format_reward": 0.8750000149011612, | |
| "step": 437 | |
| }, | |
| { | |
| "completion_length": 1237.3333892822266, | |
| "epoch": 0.5005714285714286, | |
| "grad_norm": 0.22354252636432648, | |
| "kl": 0.008880615234375, | |
| "learning_rate": 1.4150013466019114e-07, | |
| "loss": 0.0004, | |
| "reward": 0.18773654848337173, | |
| "reward_std": 0.22358889505267143, | |
| "rewards/cosine_scaled_reward": -0.07216969225555658, | |
| "rewards/format_reward": 0.9375000149011612, | |
| "step": 438 | |
| }, | |
| { | |
| "completion_length": 1286.6250305175781, | |
| "epoch": 0.5017142857142857, | |
| "grad_norm": 0.46097424626350403, | |
| "kl": 0.01509857177734375, | |
| "learning_rate": 1.4019235263722034e-07, | |
| "loss": 0.0006, | |
| "reward": 0.17482924927026033, | |
| "reward_std": 0.20090174488723278, | |
| "rewards/cosine_scaled_reward": -0.06697515025734901, | |
| "rewards/format_reward": 0.875, | |
| "step": 439 | |
| }, | |
| { | |
| "completion_length": 1781.8334045410156, | |
| "epoch": 0.5028571428571429, | |
| "grad_norm": 0.2967330515384674, | |
| "kl": 0.0180206298828125, | |
| "learning_rate": 1.3890454406082956e-07, | |
| "loss": 0.0007, | |
| "reward": 0.2402921486645937, | |
| "reward_std": 0.2830519266426563, | |
| "rewards/cosine_scaled_reward": 0.024581193923950195, | |
| "rewards/format_reward": 0.833333358168602, | |
| "step": 440 | |
| }, | |
| { | |
| "completion_length": 1721.9583740234375, | |
| "epoch": 0.504, | |
| "grad_norm": 0.28784990310668945, | |
| "kl": 0.01604461669921875, | |
| "learning_rate": 1.3763677169699217e-07, | |
| "loss": 0.0006, | |
| "reward": 0.22011593729257584, | |
| "reward_std": 0.19908079877495766, | |
| "rewards/cosine_scaled_reward": 0.01261664368212223, | |
| "rewards/format_reward": 0.8125000149011612, | |
| "step": 441 | |
| }, | |
| { | |
| "completion_length": 1423.8750610351562, | |
| "epoch": 0.5051428571428571, | |
| "grad_norm": 0.27160194516181946, | |
| "kl": 0.01059722900390625, | |
| "learning_rate": 1.3638909733514452e-07, | |
| "loss": 0.0004, | |
| "reward": 0.16875937953591347, | |
| "reward_std": 0.25345410220324993, | |
| "rewards/cosine_scaled_reward": -0.06454695202410221, | |
| "rewards/format_reward": 0.8541666716337204, | |
| "step": 442 | |
| }, | |
| { | |
| "completion_length": 1274.6250457763672, | |
| "epoch": 0.5062857142857143, | |
| "grad_norm": 0.22251670062541962, | |
| "kl": 0.00830841064453125, | |
| "learning_rate": 1.351615817851748e-07, | |
| "loss": 0.0003, | |
| "reward": 0.2590952720493078, | |
| "reward_std": 0.2291867546737194, | |
| "rewards/cosine_scaled_reward": 0.002004489302635193, | |
| "rewards/format_reward": 0.9791666716337204, | |
| "step": 443 | |
| }, | |
| { | |
| "completion_length": 1842.4583740234375, | |
| "epoch": 0.5074285714285715, | |
| "grad_norm": 0.27504411339759827, | |
| "kl": 0.01201629638671875, | |
| "learning_rate": 1.3395428487445914e-07, | |
| "loss": 0.0005, | |
| "reward": 0.3361460082232952, | |
| "reward_std": 0.22568980604410172, | |
| "rewards/cosine_scaled_reward": 0.13691299222409725, | |
| "rewards/format_reward": 0.895833358168602, | |
| "step": 444 | |
| }, | |
| { | |
| "completion_length": 1845.2500915527344, | |
| "epoch": 0.5085714285714286, | |
| "grad_norm": 0.20490147173404694, | |
| "kl": 0.01392364501953125, | |
| "learning_rate": 1.3276726544494571e-07, | |
| "loss": 0.0006, | |
| "reward": 0.24170970544219017, | |
| "reward_std": 0.23725956492125988, | |
| "rewards/cosine_scaled_reward": 0.05702884867787361, | |
| "rewards/format_reward": 0.7708333432674408, | |
| "step": 445 | |
| }, | |
| { | |
| "completion_length": 1136.5625305175781, | |
| "epoch": 0.5097142857142857, | |
| "grad_norm": 0.288634717464447, | |
| "kl": 0.00978851318359375, | |
| "learning_rate": 1.316005813502869e-07, | |
| "loss": 0.0004, | |
| "reward": 0.2657099813222885, | |
| "reward_std": 0.2308104708790779, | |
| "rewards/cosine_scaled_reward": 0.04564391728490591, | |
| "rewards/format_reward": 0.8750000149011612, | |
| "step": 446 | |
| }, | |
| { | |
| "completion_length": 1182.6041870117188, | |
| "epoch": 0.5108571428571429, | |
| "grad_norm": 0.251402884721756, | |
| "kl": 0.0087432861328125, | |
| "learning_rate": 1.3045428945301953e-07, | |
| "loss": 0.0003, | |
| "reward": 0.3128407448530197, | |
| "reward_std": 0.2510318458080292, | |
| "rewards/cosine_scaled_reward": 0.06458367872983217, | |
| "rewards/format_reward": 1.0, | |
| "step": 447 | |
| }, | |
| { | |
| "completion_length": 1315.2292175292969, | |
| "epoch": 0.512, | |
| "grad_norm": 0.22989130020141602, | |
| "kl": 0.007293701171875, | |
| "learning_rate": 1.2932844562179352e-07, | |
| "loss": 0.0003, | |
| "reward": 0.3753640539944172, | |
| "reward_std": 0.2728557586669922, | |
| "rewards/cosine_scaled_reward": 0.15566639928147197, | |
| "rewards/format_reward": 0.9791666716337204, | |
| "step": 448 | |
| }, | |
| { | |
| "completion_length": 1326.7917022705078, | |
| "epoch": 0.5131428571428571, | |
| "grad_norm": 0.28850504755973816, | |
| "kl": 0.0123291015625, | |
| "learning_rate": 1.2822310472864885e-07, | |
| "loss": 0.0005, | |
| "reward": 0.28183421678841114, | |
| "reward_std": 0.280775286257267, | |
| "rewards/cosine_scaled_reward": 0.07325122132897377, | |
| "rewards/format_reward": 0.8541666865348816, | |
| "step": 449 | |
| }, | |
| { | |
| "completion_length": 1575.5625457763672, | |
| "epoch": 0.5142857142857142, | |
| "grad_norm": 0.297370046377182, | |
| "kl": 0.01831817626953125, | |
| "learning_rate": 1.2713832064634125e-07, | |
| "loss": 0.0007, | |
| "reward": 0.19772536493837833, | |
| "reward_std": 0.20653804764151573, | |
| "rewards/cosine_scaled_reward": 0.010708175599575043, | |
| "rewards/format_reward": 0.7291666828095913, | |
| "step": 450 | |
| }, | |
| { | |
| "completion_length": 1457.0209045410156, | |
| "epoch": 0.5154285714285715, | |
| "grad_norm": 0.3056337237358093, | |
| "kl": 0.01351165771484375, | |
| "learning_rate": 1.260741462457165e-07, | |
| "loss": 0.0005, | |
| "reward": 0.2743101455271244, | |
| "reward_std": 0.32591526955366135, | |
| "rewards/cosine_scaled_reward": 0.0601138137280941, | |
| "rewards/format_reward": 0.8750000149011612, | |
| "step": 451 | |
| }, | |
| { | |
| "completion_length": 1302.5417175292969, | |
| "epoch": 0.5165714285714286, | |
| "grad_norm": 0.3352566063404083, | |
| "kl": 0.0123291015625, | |
| "learning_rate": 1.2503063339313356e-07, | |
| "loss": 0.0005, | |
| "reward": 0.19081053417176008, | |
| "reward_std": 0.3360443636775017, | |
| "rewards/cosine_scaled_reward": -0.04602043516933918, | |
| "rewards/format_reward": 0.8750000149011612, | |
| "step": 452 | |
| }, | |
| { | |
| "completion_length": 1659.9166870117188, | |
| "epoch": 0.5177142857142857, | |
| "grad_norm": 0.2861032783985138, | |
| "kl": 0.0186004638671875, | |
| "learning_rate": 1.2400783294793668e-07, | |
| "loss": 0.0007, | |
| "reward": 0.2529549226164818, | |
| "reward_std": 0.3462323397397995, | |
| "rewards/cosine_scaled_reward": 0.051917532458901405, | |
| "rewards/format_reward": 0.8125000149011612, | |
| "step": 453 | |
| }, | |
| { | |
| "completion_length": 1056.2083740234375, | |
| "epoch": 0.5188571428571429, | |
| "grad_norm": 0.23395924270153046, | |
| "kl": 0.009063720703125, | |
| "learning_rate": 1.2300579475997657e-07, | |
| "loss": 0.0004, | |
| "reward": 0.23422837536782026, | |
| "reward_std": 0.21533742547035217, | |
| "rewards/cosine_scaled_reward": -0.026755507104098797, | |
| "rewards/format_reward": 0.9791666716337204, | |
| "step": 454 | |
| }, | |
| { | |
| "completion_length": 1479.3333435058594, | |
| "epoch": 0.52, | |
| "grad_norm": 0.26352420449256897, | |
| "kl": 0.0125732421875, | |
| "learning_rate": 1.220245676671809e-07, | |
| "loss": 0.0005, | |
| "reward": 0.1705159144476056, | |
| "reward_std": 0.21411826461553574, | |
| "rewards/cosine_scaled_reward": -0.0704607660882175, | |
| "rewards/format_reward": 0.8750000149011612, | |
| "step": 455 | |
| }, | |
| { | |
| "completion_length": 1522.0209045410156, | |
| "epoch": 0.5211428571428571, | |
| "grad_norm": 0.29312658309936523, | |
| "kl": 0.01091766357421875, | |
| "learning_rate": 1.2106419949317388e-07, | |
| "loss": 0.0004, | |
| "reward": 0.24019585805945098, | |
| "reward_std": 0.14318828657269478, | |
| "rewards/cosine_scaled_reward": 0.02774641290307045, | |
| "rewards/format_reward": 0.833333358168602, | |
| "step": 456 | |
| }, | |
| { | |
| "completion_length": 1300.7084197998047, | |
| "epoch": 0.5222857142857142, | |
| "grad_norm": 0.23177440464496613, | |
| "kl": 0.0074005126953125, | |
| "learning_rate": 1.2012473704494537e-07, | |
| "loss": 0.0003, | |
| "reward": 0.2534996699541807, | |
| "reward_std": 0.2041100263595581, | |
| "rewards/cosine_scaled_reward": 0.003321884199976921, | |
| "rewards/format_reward": 0.9583333432674408, | |
| "step": 457 | |
| }, | |
| { | |
| "completion_length": 1500.3542175292969, | |
| "epoch": 0.5234285714285715, | |
| "grad_norm": 0.2277519702911377, | |
| "kl": 0.0123138427734375, | |
| "learning_rate": 1.1920622611056974e-07, | |
| "loss": 0.0005, | |
| "reward": 0.2181018777191639, | |
| "reward_std": 0.274239718914032, | |
| "rewards/cosine_scaled_reward": -0.008808528073132038, | |
| "rewards/format_reward": 0.8750000149011612, | |
| "step": 458 | |
| }, | |
| { | |
| "completion_length": 1216.4791870117188, | |
| "epoch": 0.5245714285714286, | |
| "grad_norm": 0.25994110107421875, | |
| "kl": 0.0108489990234375, | |
| "learning_rate": 1.1830871145697412e-07, | |
| "loss": 0.0004, | |
| "reward": 0.20663647167384624, | |
| "reward_std": 0.2075389288365841, | |
| "rewards/cosine_scaled_reward": -0.046165828593075275, | |
| "rewards/format_reward": 0.9375000149011612, | |
| "step": 459 | |
| }, | |
| { | |
| "completion_length": 1461.3750305175781, | |
| "epoch": 0.5257142857142857, | |
| "grad_norm": 0.27901095151901245, | |
| "kl": 0.012786865234375, | |
| "learning_rate": 1.1743223682775649e-07, | |
| "loss": 0.0005, | |
| "reward": 0.24853161070495844, | |
| "reward_std": 0.27109961211681366, | |
| "rewards/cosine_scaled_reward": 0.06439046189188957, | |
| "rewards/format_reward": 0.7708333432674408, | |
| "step": 460 | |
| }, | |
| { | |
| "completion_length": 1535.4584045410156, | |
| "epoch": 0.5268571428571428, | |
| "grad_norm": 0.2198079377412796, | |
| "kl": 0.0126800537109375, | |
| "learning_rate": 1.1657684494105386e-07, | |
| "loss": 0.0005, | |
| "reward": 0.27444493025541306, | |
| "reward_std": 0.287446316331625, | |
| "rewards/cosine_scaled_reward": 0.050167519599199295, | |
| "rewards/format_reward": 0.8958333432674408, | |
| "step": 461 | |
| }, | |
| { | |
| "completion_length": 1596.6667175292969, | |
| "epoch": 0.528, | |
| "grad_norm": 0.3223518133163452, | |
| "kl": 0.01030731201171875, | |
| "learning_rate": 1.1574257748745986e-07, | |
| "loss": 0.0004, | |
| "reward": 0.16964650806039572, | |
| "reward_std": 0.31031525135040283, | |
| "rewards/cosine_scaled_reward": -0.0880960263311863, | |
| "rewards/format_reward": 0.9166666865348816, | |
| "step": 462 | |
| }, | |
| { | |
| "completion_length": 1469.3750610351562, | |
| "epoch": 0.5291428571428571, | |
| "grad_norm": 0.33473342657089233, | |
| "kl": 0.0157318115234375, | |
| "learning_rate": 1.1492947512799328e-07, | |
| "loss": 0.0006, | |
| "reward": 0.25250214617699385, | |
| "reward_std": 0.20332858338952065, | |
| "rewards/cosine_scaled_reward": 0.07766706869006157, | |
| "rewards/format_reward": 0.7291666865348816, | |
| "step": 463 | |
| }, | |
| { | |
| "completion_length": 1470.5000305175781, | |
| "epoch": 0.5302857142857142, | |
| "grad_norm": 0.26016151905059814, | |
| "kl": 0.01175689697265625, | |
| "learning_rate": 1.1413757749211602e-07, | |
| "loss": 0.0005, | |
| "reward": 0.16552529204636812, | |
| "reward_std": 0.2517518103122711, | |
| "rewards/cosine_scaled_reward": -0.05065063759684563, | |
| "rewards/format_reward": 0.7916666716337204, | |
| "step": 464 | |
| }, | |
| { | |
| "completion_length": 1567.0000610351562, | |
| "epoch": 0.5314285714285715, | |
| "grad_norm": 0.2934165298938751, | |
| "kl": 0.0146026611328125, | |
| "learning_rate": 1.1336692317580158e-07, | |
| "loss": 0.0006, | |
| "reward": 0.2490998711436987, | |
| "reward_std": 0.24030263721942902, | |
| "rewards/cosine_scaled_reward": 0.02980668656527996, | |
| "rewards/format_reward": 0.8541666716337204, | |
| "step": 465 | |
| }, | |
| { | |
| "completion_length": 1849.2500610351562, | |
| "epoch": 0.5325714285714286, | |
| "grad_norm": 0.2325139194726944, | |
| "kl": 0.01602935791015625, | |
| "learning_rate": 1.1261754973965422e-07, | |
| "loss": 0.0006, | |
| "reward": 0.07940827216953039, | |
| "reward_std": 0.18165934830904007, | |
| "rewards/cosine_scaled_reward": -0.17715821415185928, | |
| "rewards/format_reward": 0.8333333432674408, | |
| "step": 466 | |
| }, | |
| { | |
| "completion_length": 1414.5625610351562, | |
| "epoch": 0.5337142857142857, | |
| "grad_norm": 0.2346559762954712, | |
| "kl": 0.00949859619140625, | |
| "learning_rate": 1.1188949370707787e-07, | |
| "loss": 0.0004, | |
| "reward": 0.21573650650680065, | |
| "reward_std": 0.20873811468482018, | |
| "rewards/cosine_scaled_reward": -0.05660676956176758, | |
| "rewards/format_reward": 1.0, | |
| "step": 467 | |
| }, | |
| { | |
| "completion_length": 1912.9376220703125, | |
| "epoch": 0.5348571428571428, | |
| "grad_norm": 0.3037513196468353, | |
| "kl": 0.01833343505859375, | |
| "learning_rate": 1.1118279056249653e-07, | |
| "loss": 0.0007, | |
| "reward": 0.2879499550908804, | |
| "reward_std": 0.28192097321152687, | |
| "rewards/cosine_scaled_reward": 0.11339114978909492, | |
| "rewards/format_reward": 0.7708333432674408, | |
| "step": 468 | |
| }, | |
| { | |
| "completion_length": 1271.7083587646484, | |
| "epoch": 0.536, | |
| "grad_norm": 0.3204426169395447, | |
| "kl": 0.014739990234375, | |
| "learning_rate": 1.1049747474962444e-07, | |
| "loss": 0.0006, | |
| "reward": 0.24237569607794285, | |
| "reward_std": 0.25575844570994377, | |
| "rewards/cosine_scaled_reward": 0.004833955317735672, | |
| "rewards/format_reward": 0.9166666716337204, | |
| "step": 469 | |
| }, | |
| { | |
| "completion_length": 1590.3125610351562, | |
| "epoch": 0.5371428571428571, | |
| "grad_norm": 0.2925020456314087, | |
| "kl": 0.0179595947265625, | |
| "learning_rate": 1.0983357966978745e-07, | |
| "loss": 0.0007, | |
| "reward": 0.3055003173649311, | |
| "reward_std": 0.19016432389616966, | |
| "rewards/cosine_scaled_reward": 0.11489966697990894, | |
| "rewards/format_reward": 0.8333333432674408, | |
| "step": 470 | |
| }, | |
| { | |
| "completion_length": 1118.3958587646484, | |
| "epoch": 0.5382857142857143, | |
| "grad_norm": 0.2672974467277527, | |
| "kl": 0.01080322265625, | |
| "learning_rate": 1.0919113768029517e-07, | |
| "loss": 0.0004, | |
| "reward": 0.05892130918800831, | |
| "reward_std": 0.20794951915740967, | |
| "rewards/cosine_scaled_reward": -0.2085135094821453, | |
| "rewards/format_reward": 0.8541666865348816, | |
| "step": 471 | |
| }, | |
| { | |
| "completion_length": 906.6458587646484, | |
| "epoch": 0.5394285714285715, | |
| "grad_norm": 0.2568436861038208, | |
| "kl": 0.0084228515625, | |
| "learning_rate": 1.0857018009286381e-07, | |
| "loss": 0.0003, | |
| "reward": 0.2956396248191595, | |
| "reward_std": 0.2609308622777462, | |
| "rewards/cosine_scaled_reward": 0.06847019493579865, | |
| "rewards/format_reward": 0.9375, | |
| "step": 472 | |
| }, | |
| { | |
| "completion_length": 1430.8333740234375, | |
| "epoch": 0.5405714285714286, | |
| "grad_norm": 0.30554234981536865, | |
| "kl": 0.01314544677734375, | |
| "learning_rate": 1.0797073717209013e-07, | |
| "loss": 0.0005, | |
| "reward": 0.05116685951361433, | |
| "reward_std": 0.2041519582271576, | |
| "rewards/cosine_scaled_reward": -0.20498012378811836, | |
| "rewards/format_reward": 0.8125000149011612, | |
| "step": 473 | |
| }, | |
| { | |
| "completion_length": 1448.8333587646484, | |
| "epoch": 0.5417142857142857, | |
| "grad_norm": 0.33508339524269104, | |
| "kl": 0.00983428955078125, | |
| "learning_rate": 1.0739283813397639e-07, | |
| "loss": 0.0004, | |
| "reward": 0.35246733389794827, | |
| "reward_std": 0.2351733185350895, | |
| "rewards/cosine_scaled_reward": 0.12773746624588966, | |
| "rewards/format_reward": 0.9791666716337204, | |
| "step": 474 | |
| }, | |
| { | |
| "completion_length": 1581.5834045410156, | |
| "epoch": 0.5428571428571428, | |
| "grad_norm": 0.42422112822532654, | |
| "kl": 0.016143798828125, | |
| "learning_rate": 1.068365111445064e-07, | |
| "loss": 0.0006, | |
| "reward": 0.22443121299147606, | |
| "reward_std": 0.31280872225761414, | |
| "rewards/cosine_scaled_reward": -0.00971287488937378, | |
| "rewards/format_reward": 0.8750000149011612, | |
| "step": 475 | |
| }, | |
| { | |
| "completion_length": 1366.2500457763672, | |
| "epoch": 0.544, | |
| "grad_norm": 0.3073312044143677, | |
| "kl": 0.0132598876953125, | |
| "learning_rate": 1.063017833182728e-07, | |
| "loss": 0.0005, | |
| "reward": 0.29938487708568573, | |
| "reward_std": 0.14425114821642637, | |
| "rewards/cosine_scaled_reward": 0.09217657893896103, | |
| "rewards/format_reward": 0.8750000149011612, | |
| "step": 476 | |
| }, | |
| { | |
| "completion_length": 965.2708435058594, | |
| "epoch": 0.5451428571428572, | |
| "grad_norm": 0.30544164776802063, | |
| "kl": 0.0073089599609375, | |
| "learning_rate": 1.0578868071715544e-07, | |
| "loss": 0.0003, | |
| "reward": 0.39359666407108307, | |
| "reward_std": 0.25950491055846214, | |
| "rewards/cosine_scaled_reward": 0.17642678815172985, | |
| "rewards/format_reward": 0.9791666716337204, | |
| "step": 477 | |
| }, | |
| { | |
| "completion_length": 1636.5209045410156, | |
| "epoch": 0.5462857142857143, | |
| "grad_norm": 0.2438599169254303, | |
| "kl": 0.011138916015625, | |
| "learning_rate": 1.0529722834905125e-07, | |
| "loss": 0.0004, | |
| "reward": 0.1690983809530735, | |
| "reward_std": 0.265783216804266, | |
| "rewards/cosine_scaled_reward": -0.06696726009249687, | |
| "rewards/format_reward": 0.8541666865348816, | |
| "step": 478 | |
| }, | |
| { | |
| "completion_length": 1457.8333740234375, | |
| "epoch": 0.5474285714285714, | |
| "grad_norm": 0.36753106117248535, | |
| "kl": 0.02089691162109375, | |
| "learning_rate": 1.0482745016665526e-07, | |
| "loss": 0.0008, | |
| "reward": 0.24042591266334057, | |
| "reward_std": 0.28960849717259407, | |
| "rewards/cosine_scaled_reward": 0.03149300068616867, | |
| "rewards/format_reward": 0.833333358168602, | |
| "step": 479 | |
| }, | |
| { | |
| "completion_length": 1351.2916870117188, | |
| "epoch": 0.5485714285714286, | |
| "grad_norm": 0.18641342222690582, | |
| "kl": 0.00542449951171875, | |
| "learning_rate": 1.0437936906629334e-07, | |
| "loss": 0.0002, | |
| "reward": 0.35221418365836143, | |
| "reward_std": 0.2596045136451721, | |
| "rewards/cosine_scaled_reward": 0.13672377169132233, | |
| "rewards/format_reward": 0.9583333432674408, | |
| "step": 480 | |
| }, | |
| { | |
| "completion_length": 1663.6458587646484, | |
| "epoch": 0.5497142857142857, | |
| "grad_norm": 0.2585221230983734, | |
| "kl": 0.019439697265625, | |
| "learning_rate": 1.0395300688680625e-07, | |
| "loss": 0.0008, | |
| "reward": 0.08225089451298118, | |
| "reward_std": 0.24548032693564892, | |
| "rewards/cosine_scaled_reward": -0.11919093690812588, | |
| "rewards/format_reward": 0.6666666716337204, | |
| "step": 481 | |
| }, | |
| { | |
| "completion_length": 1130.1250305175781, | |
| "epoch": 0.5508571428571428, | |
| "grad_norm": 0.30442705750465393, | |
| "kl": 0.0112762451171875, | |
| "learning_rate": 1.0354838440848501e-07, | |
| "loss": 0.0005, | |
| "reward": 0.2443934567272663, | |
| "reward_std": 0.24509599804878235, | |
| "rewards/cosine_scaled_reward": -0.007126476150006056, | |
| "rewards/format_reward": 0.9583333432674408, | |
| "step": 482 | |
| }, | |
| { | |
| "completion_length": 1528.354248046875, | |
| "epoch": 0.552, | |
| "grad_norm": 0.4626905024051666, | |
| "kl": 0.0183868408203125, | |
| "learning_rate": 1.0316552135205837e-07, | |
| "loss": 0.0007, | |
| "reward": 0.23332804813981056, | |
| "reward_std": 0.3575605973601341, | |
| "rewards/cosine_scaled_reward": 0.03512163204140961, | |
| "rewards/format_reward": 0.7916666716337204, | |
| "step": 483 | |
| }, | |
| { | |
| "completion_length": 1363.7500610351562, | |
| "epoch": 0.5531428571428572, | |
| "grad_norm": 0.30249202251434326, | |
| "kl": 0.01682281494140625, | |
| "learning_rate": 1.0280443637773163e-07, | |
| "loss": 0.0007, | |
| "reward": 0.35776749555952847, | |
| "reward_std": 0.2735111825168133, | |
| "rewards/cosine_scaled_reward": 0.18149008601903915, | |
| "rewards/format_reward": 0.833333358168602, | |
| "step": 484 | |
| }, | |
| { | |
| "completion_length": 1521.4584350585938, | |
| "epoch": 0.5542857142857143, | |
| "grad_norm": 0.32728955149650574, | |
| "kl": 0.017608642578125, | |
| "learning_rate": 1.0246514708427701e-07, | |
| "loss": 0.0007, | |
| "reward": 0.2742725261487067, | |
| "reward_std": 0.23011274076998234, | |
| "rewards/cosine_scaled_reward": 0.037165215238928795, | |
| "rewards/format_reward": 0.9375000149011612, | |
| "step": 485 | |
| }, | |
| { | |
| "completion_length": 1431.0625305175781, | |
| "epoch": 0.5554285714285714, | |
| "grad_norm": 0.3946894407272339, | |
| "kl": 0.01422882080078125, | |
| "learning_rate": 1.0214767000817596e-07, | |
| "loss": 0.0006, | |
| "reward": 0.3760902900248766, | |
| "reward_std": 0.2089155912399292, | |
| "rewards/cosine_scaled_reward": 0.1899992600083351, | |
| "rewards/format_reward": 0.8958333432674408, | |
| "step": 486 | |
| }, | |
| { | |
| "completion_length": 1597.7708435058594, | |
| "epoch": 0.5565714285714286, | |
| "grad_norm": 0.2671118974685669, | |
| "kl": 0.01332855224609375, | |
| "learning_rate": 1.0185202062281336e-07, | |
| "loss": 0.0005, | |
| "reward": 0.22492020577192307, | |
| "reward_std": 0.26971185952425003, | |
| "rewards/cosine_scaled_reward": -0.019520113710314035, | |
| "rewards/format_reward": 0.9166666865348816, | |
| "step": 487 | |
| }, | |
| { | |
| "completion_length": 1713.3333740234375, | |
| "epoch": 0.5577142857142857, | |
| "grad_norm": 0.2240956574678421, | |
| "kl": 0.01678466796875, | |
| "learning_rate": 1.0157821333772304e-07, | |
| "loss": 0.0007, | |
| "reward": 0.13526278641074896, | |
| "reward_std": 0.2893335670232773, | |
| "rewards/cosine_scaled_reward": -0.10202349070459604, | |
| "rewards/format_reward": 0.8333333432674408, | |
| "step": 488 | |
| }, | |
| { | |
| "completion_length": 1029.1875381469727, | |
| "epoch": 0.5588571428571428, | |
| "grad_norm": 0.31260091066360474, | |
| "kl": 0.01323699951171875, | |
| "learning_rate": 1.013262614978859e-07, | |
| "loss": 0.0005, | |
| "reward": 0.4309612764045596, | |
| "reward_std": 0.253094045445323, | |
| "rewards/cosine_scaled_reward": 0.2355925627052784, | |
| "rewards/format_reward": 0.9375000149011612, | |
| "step": 489 | |
| }, | |
| { | |
| "completion_length": 1255.187515258789, | |
| "epoch": 0.56, | |
| "grad_norm": 0.3444691300392151, | |
| "kl": 0.01079559326171875, | |
| "learning_rate": 1.0109617738307911e-07, | |
| "loss": 0.0004, | |
| "reward": 0.2048361971974373, | |
| "reward_std": 0.21078473329544067, | |
| "rewards/cosine_scaled_reward": -0.05001578852534294, | |
| "rewards/format_reward": 0.9375000149011612, | |
| "step": 490 | |
| }, | |
| { | |
| "completion_length": 1938.979248046875, | |
| "epoch": 0.5611428571428572, | |
| "grad_norm": 0.29255354404449463, | |
| "kl": 0.0247802734375, | |
| "learning_rate": 1.0088797220727779e-07, | |
| "loss": 0.001, | |
| "reward": 0.21013529412448406, | |
| "reward_std": 0.19084220007061958, | |
| "rewards/cosine_scaled_reward": 0.013043895363807678, | |
| "rewards/format_reward": 0.7708333432674408, | |
| "step": 491 | |
| }, | |
| { | |
| "completion_length": 1133.6875457763672, | |
| "epoch": 0.5622857142857143, | |
| "grad_norm": 0.4209684431552887, | |
| "kl": 0.0138702392578125, | |
| "learning_rate": 1.0070165611810855e-07, | |
| "loss": 0.0006, | |
| "reward": 0.16102333180606365, | |
| "reward_std": 0.17116187885403633, | |
| "rewards/cosine_scaled_reward": -0.0879567414522171, | |
| "rewards/format_reward": 0.8750000149011612, | |
| "step": 492 | |
| }, | |
| { | |
| "completion_length": 1594.7083740234375, | |
| "epoch": 0.5634285714285714, | |
| "grad_norm": 0.31390249729156494, | |
| "kl": 0.0111083984375, | |
| "learning_rate": 1.005372381963547e-07, | |
| "loss": 0.0004, | |
| "reward": 0.16091343411244452, | |
| "reward_std": 0.22968443483114243, | |
| "rewards/cosine_scaled_reward": -0.09001005440950394, | |
| "rewards/format_reward": 0.8958333432674408, | |
| "step": 493 | |
| }, | |
| { | |
| "completion_length": 1279.25, | |
| "epoch": 0.5645714285714286, | |
| "grad_norm": 0.33988407254219055, | |
| "kl": 0.01303863525390625, | |
| "learning_rate": 1.0039472645551372e-07, | |
| "loss": 0.0005, | |
| "reward": 0.11660805623978376, | |
| "reward_std": 0.20437689498066902, | |
| "rewards/cosine_scaled_reward": -0.14967265352606773, | |
| "rewards/format_reward": 0.8958333432674408, | |
| "step": 494 | |
| }, | |
| { | |
| "completion_length": 1453.0209045410156, | |
| "epoch": 0.5657142857142857, | |
| "grad_norm": 0.43056032061576843, | |
| "kl": 0.0152130126953125, | |
| "learning_rate": 1.002741278414069e-07, | |
| "loss": 0.0006, | |
| "reward": 0.2999148261733353, | |
| "reward_std": 0.21086948364973068, | |
| "rewards/cosine_scaled_reward": 0.07477198913693428, | |
| "rewards/format_reward": 0.9375000149011612, | |
| "step": 495 | |
| }, | |
| { | |
| "completion_length": 1207.75, | |
| "epoch": 0.5668571428571428, | |
| "grad_norm": 0.28147271275520325, | |
| "kl": 0.01139068603515625, | |
| "learning_rate": 1.0017544823184055e-07, | |
| "loss": 0.0005, | |
| "reward": 0.37165018916130066, | |
| "reward_std": 0.2604399360716343, | |
| "rewards/cosine_scaled_reward": 0.16403314599301666, | |
| "rewards/format_reward": 0.9166666865348816, | |
| "step": 496 | |
| }, | |
| { | |
| "completion_length": 1213.7708435058594, | |
| "epoch": 0.568, | |
| "grad_norm": 0.36225390434265137, | |
| "kl": 0.0146331787109375, | |
| "learning_rate": 1.0009869243631952e-07, | |
| "loss": 0.0006, | |
| "reward": 0.41349948197603226, | |
| "reward_std": 0.18646394088864326, | |
| "rewards/cosine_scaled_reward": 0.23145748116075993, | |
| "rewards/format_reward": 0.895833358168602, | |
| "step": 497 | |
| }, | |
| { | |
| "completion_length": 1448.729232788086, | |
| "epoch": 0.5691428571428572, | |
| "grad_norm": 0.3582679331302643, | |
| "kl": 0.015716552734375, | |
| "learning_rate": 1.000438641958131e-07, | |
| "loss": 0.0006, | |
| "reward": 0.09725892543792725, | |
| "reward_std": 0.24059875309467316, | |
| "rewards/cosine_scaled_reward": -0.1641225963830948, | |
| "rewards/format_reward": 0.8541666865348816, | |
| "step": 498 | |
| }, | |
| { | |
| "completion_length": 1700.8959045410156, | |
| "epoch": 0.5702857142857143, | |
| "grad_norm": 0.2774902582168579, | |
| "kl": 0.01750946044921875, | |
| "learning_rate": 1.0001096618257236e-07, | |
| "loss": 0.0007, | |
| "reward": 0.11731046251952648, | |
| "reward_std": 0.2501941677182913, | |
| "rewards/cosine_scaled_reward": -0.11795766558498144, | |
| "rewards/format_reward": 0.8125000149011612, | |
| "step": 499 | |
| }, | |
| { | |
| "completion_length": 1141.7917022705078, | |
| "epoch": 0.5714285714285714, | |
| "grad_norm": 0.32746848464012146, | |
| "kl": 0.0137176513671875, | |
| "learning_rate": 1e-07, | |
| "loss": 0.0005, | |
| "reward": 0.23398451320827007, | |
| "reward_std": 0.2770588528364897, | |
| "rewards/cosine_scaled_reward": 0.03950834833085537, | |
| "rewards/format_reward": 0.7916666865348816, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 0.5714285714285714, | |
| "step": 500, | |
| "total_flos": 0.0, | |
| "train_loss": 0.00026818006123357916, | |
| "train_runtime": 27510.2217, | |
| "train_samples_per_second": 0.872, | |
| "train_steps_per_second": 0.018 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 500, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 50, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 4, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |