| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.3931203931203931, | |
| "eval_steps": 500, | |
| "global_step": 200, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 562.125, | |
| "epoch": 0.0019656019656019656, | |
| "grad_norm": 0.0, | |
| "kl": 0.0, | |
| "learning_rate": 5e-08, | |
| "loss": 0.0, | |
| "num_tokens": 22458.0, | |
| "reward": 0.0, | |
| "reward_std": 0.0, | |
| "rewards/reward_func": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 382.84375, | |
| "epoch": 0.003931203931203931, | |
| "grad_norm": 0.0, | |
| "kl": 0.0, | |
| "learning_rate": 1e-07, | |
| "loss": 0.0, | |
| "num_tokens": 39539.0, | |
| "reward": 0.0, | |
| "reward_std": 0.0, | |
| "rewards/reward_func": 0.0, | |
| "step": 2 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 296.84375, | |
| "epoch": 0.005896805896805897, | |
| "grad_norm": 0.0, | |
| "kl": 0.0, | |
| "learning_rate": 1.5e-07, | |
| "loss": 0.0, | |
| "num_tokens": 52892.0, | |
| "reward": 0.0, | |
| "reward_std": 0.0, | |
| "rewards/reward_func": 0.0, | |
| "step": 3 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 562.84375, | |
| "epoch": 0.007862407862407862, | |
| "grad_norm": 0.0, | |
| "kl": 0.0, | |
| "learning_rate": 2e-07, | |
| "loss": 0.0, | |
| "num_tokens": 73799.0, | |
| "reward": 0.0, | |
| "reward_std": 0.0, | |
| "rewards/reward_func": 0.0, | |
| "step": 4 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 454.9375, | |
| "epoch": 0.009828009828009828, | |
| "grad_norm": 0.0, | |
| "kl": 0.0, | |
| "learning_rate": 2.5e-07, | |
| "loss": 0.0, | |
| "num_tokens": 92389.0, | |
| "reward": 0.0, | |
| "reward_std": 0.0, | |
| "rewards/reward_func": 0.0, | |
| "step": 5 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 401.5625, | |
| "epoch": 0.011793611793611793, | |
| "grad_norm": 0.8271026359027706, | |
| "kl": 0.0, | |
| "learning_rate": 3e-07, | |
| "loss": 0.0459, | |
| "num_tokens": 108305.0, | |
| "reward": 0.03125, | |
| "reward_std": 0.04419417306780815, | |
| "rewards/reward_func": 0.03125, | |
| "step": 6 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 390.625, | |
| "epoch": 0.013759213759213759, | |
| "grad_norm": 0.0003446225536382493, | |
| "kl": 0.0002499818801879883, | |
| "learning_rate": 3.5e-07, | |
| "loss": 0.0, | |
| "num_tokens": 125607.0, | |
| "reward": 0.0, | |
| "reward_std": 0.0, | |
| "rewards/reward_func": 0.0, | |
| "step": 7 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 448.75, | |
| "epoch": 0.015724815724815724, | |
| "grad_norm": 0.0013026212429223525, | |
| "kl": 0.0004665851593017578, | |
| "learning_rate": 4e-07, | |
| "loss": 0.0, | |
| "num_tokens": 142955.0, | |
| "reward": 0.0, | |
| "reward_std": 0.0, | |
| "rewards/reward_func": 0.0, | |
| "step": 8 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 461.90625, | |
| "epoch": 0.01769041769041769, | |
| "grad_norm": 0.0021171088963702245, | |
| "kl": 0.0006817877292633057, | |
| "learning_rate": 4.5e-07, | |
| "loss": 0.0, | |
| "num_tokens": 162956.0, | |
| "reward": 0.0, | |
| "reward_std": 0.0, | |
| "rewards/reward_func": 0.0, | |
| "step": 9 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 330.6875, | |
| "epoch": 0.019656019656019656, | |
| "grad_norm": 1.3023849518044601, | |
| "kl": 0.0006229877471923828, | |
| "learning_rate": 5e-07, | |
| "loss": 0.0444, | |
| "num_tokens": 177758.0, | |
| "reward": 0.0625, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/reward_func": 0.0625, | |
| "step": 10 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 513.1875, | |
| "epoch": 0.021621621621621623, | |
| "grad_norm": 0.00027502974416413834, | |
| "kl": 0.00018683075904846191, | |
| "learning_rate": 5.5e-07, | |
| "loss": 0.0, | |
| "num_tokens": 197364.0, | |
| "reward": 0.0, | |
| "reward_std": 0.0, | |
| "rewards/reward_func": 0.0, | |
| "step": 11 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 424.625, | |
| "epoch": 0.023587223587223587, | |
| "grad_norm": 0.6170315097653724, | |
| "kl": 0.00029702484607696533, | |
| "learning_rate": 6e-07, | |
| "loss": -0.0489, | |
| "num_tokens": 214156.0, | |
| "reward": 0.03125, | |
| "reward_std": 0.04419417306780815, | |
| "rewards/reward_func": 0.03125, | |
| "step": 12 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 532.1875, | |
| "epoch": 0.025552825552825554, | |
| "grad_norm": 0.0002590285678817397, | |
| "kl": 0.00019553303718566895, | |
| "learning_rate": 6.5e-07, | |
| "loss": 0.0, | |
| "num_tokens": 234938.0, | |
| "reward": 0.0, | |
| "reward_std": 0.0, | |
| "rewards/reward_func": 0.0, | |
| "step": 13 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 391.6875, | |
| "epoch": 0.027518427518427518, | |
| "grad_norm": 0.2468481109530296, | |
| "kl": 0.0002957582473754883, | |
| "learning_rate": 7e-07, | |
| "loss": -0.042, | |
| "num_tokens": 250970.0, | |
| "reward": 0.03125, | |
| "reward_std": 0.04419417306780815, | |
| "rewards/reward_func": 0.03125, | |
| "step": 14 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 526.75, | |
| "epoch": 0.029484029484029485, | |
| "grad_norm": 0.6658783590054981, | |
| "kl": 0.00033867359161376953, | |
| "learning_rate": 7.5e-07, | |
| "loss": -0.0001, | |
| "num_tokens": 271842.0, | |
| "reward": 0.03125, | |
| "reward_std": 0.04419417306780815, | |
| "rewards/reward_func": 0.03125, | |
| "step": 15 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 357.5, | |
| "epoch": 0.03144963144963145, | |
| "grad_norm": 0.001449294133896928, | |
| "kl": 0.00034862756729125977, | |
| "learning_rate": 8e-07, | |
| "loss": 0.0, | |
| "num_tokens": 288920.0, | |
| "reward": 0.0, | |
| "reward_std": 0.0, | |
| "rewards/reward_func": 0.0, | |
| "step": 16 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 466.9375, | |
| "epoch": 0.03341523341523341, | |
| "grad_norm": 0.000550624611359764, | |
| "kl": 0.00036454200744628906, | |
| "learning_rate": 8.499999999999999e-07, | |
| "loss": 0.0, | |
| "num_tokens": 306990.0, | |
| "reward": 0.0, | |
| "reward_std": 0.0, | |
| "rewards/reward_func": 0.0, | |
| "step": 17 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 425.15625, | |
| "epoch": 0.03538083538083538, | |
| "grad_norm": 0.4966139299418229, | |
| "kl": 0.0004194974899291992, | |
| "learning_rate": 9e-07, | |
| "loss": -0.059, | |
| "num_tokens": 324483.0, | |
| "reward": 0.03125, | |
| "reward_std": 0.04419417306780815, | |
| "rewards/reward_func": 0.03125, | |
| "step": 18 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 558.84375, | |
| "epoch": 0.03734643734643735, | |
| "grad_norm": 0.0005133453694281673, | |
| "kl": 0.000286102294921875, | |
| "learning_rate": 9.499999999999999e-07, | |
| "loss": 0.0, | |
| "num_tokens": 346726.0, | |
| "reward": 0.0, | |
| "reward_std": 0.0, | |
| "rewards/reward_func": 0.0, | |
| "step": 19 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 462.96875, | |
| "epoch": 0.03931203931203931, | |
| "grad_norm": 0.7524081119290347, | |
| "kl": 0.0004723072052001953, | |
| "learning_rate": 1e-06, | |
| "loss": -0.051, | |
| "num_tokens": 364915.0, | |
| "reward": 0.0625, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/reward_func": 0.0625, | |
| "step": 20 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 442.84375, | |
| "epoch": 0.041277641277641275, | |
| "grad_norm": 0.002276209851987083, | |
| "kl": 0.00047028064727783203, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0, | |
| "num_tokens": 382538.0, | |
| "reward": 0.0, | |
| "reward_std": 0.0, | |
| "rewards/reward_func": 0.0, | |
| "step": 21 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 487.8125, | |
| "epoch": 0.043243243243243246, | |
| "grad_norm": 0.0009159776748314612, | |
| "kl": 0.0003075599670410156, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0, | |
| "num_tokens": 402660.0, | |
| "reward": 0.0, | |
| "reward_std": 0.0, | |
| "rewards/reward_func": 0.0, | |
| "step": 22 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 521.09375, | |
| "epoch": 0.04520884520884521, | |
| "grad_norm": 0.4431344945845042, | |
| "kl": 0.0008206367492675781, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0654, | |
| "num_tokens": 423169.0, | |
| "reward": 0.03125, | |
| "reward_std": 0.04419417306780815, | |
| "rewards/reward_func": 0.03125, | |
| "step": 23 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 424.9375, | |
| "epoch": 0.04717444717444717, | |
| "grad_norm": 0.00201223494239014, | |
| "kl": 0.0007357597351074219, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0, | |
| "num_tokens": 439839.0, | |
| "reward": 0.0625, | |
| "reward_std": 0.0, | |
| "rewards/reward_func": 0.0625, | |
| "step": 24 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 593.625, | |
| "epoch": 0.04914004914004914, | |
| "grad_norm": 0.18561111784701523, | |
| "kl": 0.0019412040710449219, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0044, | |
| "num_tokens": 463265.0, | |
| "reward": 0.09375, | |
| "reward_std": 0.04419417306780815, | |
| "rewards/reward_func": 0.09375, | |
| "step": 25 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 280.375, | |
| "epoch": 0.05110565110565111, | |
| "grad_norm": 5.884242558131956, | |
| "kl": 0.0025135278701782227, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0569, | |
| "num_tokens": 475453.0, | |
| "reward": 0.03125, | |
| "reward_std": 0.04419417306780815, | |
| "rewards/reward_func": 0.03125, | |
| "step": 26 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 513.90625, | |
| "epoch": 0.05307125307125307, | |
| "grad_norm": 0.0019098622203968013, | |
| "kl": 0.0006109476089477539, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0, | |
| "num_tokens": 495570.0, | |
| "reward": 0.0, | |
| "reward_std": 0.0, | |
| "rewards/reward_func": 0.0, | |
| "step": 27 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 537.8125, | |
| "epoch": 0.055036855036855035, | |
| "grad_norm": 0.22075214354309136, | |
| "kl": 0.0009493827819824219, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0333, | |
| "num_tokens": 516232.0, | |
| "reward": 0.03125, | |
| "reward_std": 0.04419417306780815, | |
| "rewards/reward_func": 0.03125, | |
| "step": 28 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 485.6875, | |
| "epoch": 0.057002457002457, | |
| "grad_norm": 0.0027609914881789005, | |
| "kl": 0.0013211965560913086, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0, | |
| "num_tokens": 534828.0, | |
| "reward": 0.0, | |
| "reward_std": 0.0, | |
| "rewards/reward_func": 0.0, | |
| "step": 29 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 394.8125, | |
| "epoch": 0.05896805896805897, | |
| "grad_norm": 1035336.9697758019, | |
| "kl": 7488.001363635063, | |
| "learning_rate": 1e-06, | |
| "loss": 7.5002, | |
| "num_tokens": 550682.0, | |
| "reward": 0.03125, | |
| "reward_std": 0.04419417306780815, | |
| "rewards/reward_func": 0.03125, | |
| "step": 30 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 472.96875, | |
| "epoch": 0.060933660933660934, | |
| "grad_norm": 0.005787911590971138, | |
| "kl": 0.002418994903564453, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0, | |
| "num_tokens": 568901.0, | |
| "reward": 0.0, | |
| "reward_std": 0.0, | |
| "rewards/reward_func": 0.0, | |
| "step": 31 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 532.1875, | |
| "epoch": 0.0628992628992629, | |
| "grad_norm": 0.22606153101175416, | |
| "kl": 0.0019965171813964844, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0, | |
| "num_tokens": 589439.0, | |
| "reward": 0.03125, | |
| "reward_std": 0.04419417306780815, | |
| "rewards/reward_func": 0.03125, | |
| "step": 32 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 443.78125, | |
| "epoch": 0.06486486486486487, | |
| "grad_norm": 0.35741687124698773, | |
| "kl": 0.002090930938720703, | |
| "learning_rate": 1e-06, | |
| "loss": 0.007, | |
| "num_tokens": 606976.0, | |
| "reward": 0.03125, | |
| "reward_std": 0.04419417306780815, | |
| "rewards/reward_func": 0.03125, | |
| "step": 33 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 582.25, | |
| "epoch": 0.06683046683046683, | |
| "grad_norm": 0.0012562903075668153, | |
| "kl": 0.0010848045349121094, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0, | |
| "num_tokens": 629146.0, | |
| "reward": 0.0, | |
| "reward_std": 0.0, | |
| "rewards/reward_func": 0.0, | |
| "step": 34 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 445.96875, | |
| "epoch": 0.0687960687960688, | |
| "grad_norm": 0.9035998758815753, | |
| "kl": 0.0037384033203125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0705, | |
| "num_tokens": 646643.0, | |
| "reward": 0.125, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/reward_func": 0.125, | |
| "step": 35 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 444.6875, | |
| "epoch": 0.07076167076167077, | |
| "grad_norm": 0.3422524583969719, | |
| "kl": 0.0064411163330078125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0749, | |
| "num_tokens": 666223.0, | |
| "reward": 0.03125, | |
| "reward_std": 0.04419417306780815, | |
| "rewards/reward_func": 0.03125, | |
| "step": 36 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 375.15625, | |
| "epoch": 0.07272727272727272, | |
| "grad_norm": 0.6543309959643355, | |
| "kl": 0.025150299072265625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.074, | |
| "num_tokens": 681430.0, | |
| "reward": 0.03125, | |
| "reward_std": 0.04419417306780815, | |
| "rewards/reward_func": 0.03125, | |
| "step": 37 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 465.40625, | |
| "epoch": 0.0746928746928747, | |
| "grad_norm": 0.4629932120108371, | |
| "kl": 0.006771087646484375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0, | |
| "num_tokens": 699985.0, | |
| "reward": 0.09375, | |
| "reward_std": 0.04419417306780815, | |
| "rewards/reward_func": 0.09375, | |
| "step": 38 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 386.46875, | |
| "epoch": 0.07665847665847667, | |
| "grad_norm": 0.2536795642942375, | |
| "kl": 0.010662078857421875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0128, | |
| "num_tokens": 715666.0, | |
| "reward": 0.03125, | |
| "reward_std": 0.04419417306780815, | |
| "rewards/reward_func": 0.03125, | |
| "step": 39 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 398.90625, | |
| "epoch": 0.07862407862407862, | |
| "grad_norm": 0.36055266191548657, | |
| "kl": 0.014057159423828125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0222, | |
| "num_tokens": 732739.0, | |
| "reward": 0.03125, | |
| "reward_std": 0.04419417306780815, | |
| "rewards/reward_func": 0.03125, | |
| "step": 40 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 497.78125, | |
| "epoch": 0.08058968058968059, | |
| "grad_norm": 0.5439508463238408, | |
| "kl": 0.011962890625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0736, | |
| "num_tokens": 752036.0, | |
| "reward": 0.0625, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/reward_func": 0.0625, | |
| "step": 41 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 553.9375, | |
| "epoch": 0.08255528255528255, | |
| "grad_norm": 0.2940020586639228, | |
| "kl": 0.01055145263671875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0098, | |
| "num_tokens": 774304.0, | |
| "reward": 0.03125, | |
| "reward_std": 0.04419417306780815, | |
| "rewards/reward_func": 0.03125, | |
| "step": 42 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 506.78125, | |
| "epoch": 0.08452088452088452, | |
| "grad_norm": 0.34987523935436315, | |
| "kl": 0.022228240966796875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0136, | |
| "num_tokens": 794573.0, | |
| "reward": 0.03125, | |
| "reward_std": 0.04419417306780815, | |
| "rewards/reward_func": 0.03125, | |
| "step": 43 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 534.71875, | |
| "epoch": 0.08648648648648649, | |
| "grad_norm": 0.28775109044810143, | |
| "kl": 0.016704559326171875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.022, | |
| "num_tokens": 814952.0, | |
| "reward": 0.0625, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/reward_func": 0.0625, | |
| "step": 44 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 551.125, | |
| "epoch": 0.08845208845208845, | |
| "grad_norm": 0.5294418172877363, | |
| "kl": 0.01171875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0127, | |
| "num_tokens": 837090.0, | |
| "reward": 0.0625, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/reward_func": 0.0625, | |
| "step": 45 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 424.09375, | |
| "epoch": 0.09041769041769042, | |
| "grad_norm": 0.7442762340799661, | |
| "kl": 0.0166015625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0527, | |
| "num_tokens": 855063.0, | |
| "reward": 0.28125, | |
| "reward_std": 0.13258251920342445, | |
| "rewards/reward_func": 0.28125, | |
| "step": 46 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 578.90625, | |
| "epoch": 0.09238329238329239, | |
| "grad_norm": 0.9964430347064654, | |
| "kl": 0.01439666748046875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0347, | |
| "num_tokens": 876960.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.2651650384068489, | |
| "rewards/reward_func": 0.1875, | |
| "step": 47 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 516.65625, | |
| "epoch": 0.09434889434889435, | |
| "grad_norm": 0.0029736329333483326, | |
| "kl": 0.015178680419921875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0, | |
| "num_tokens": 897477.0, | |
| "reward": 0.0, | |
| "reward_std": 0.0, | |
| "rewards/reward_func": 0.0, | |
| "step": 48 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 406.53125, | |
| "epoch": 0.09631449631449632, | |
| "grad_norm": 0.42366305164369583, | |
| "kl": 0.0214385986328125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.023, | |
| "num_tokens": 913848.0, | |
| "reward": 0.0625, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/reward_func": 0.0625, | |
| "step": 49 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 615.46875, | |
| "epoch": 0.09828009828009827, | |
| "grad_norm": 0.2894821606425995, | |
| "kl": 0.010334014892578125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0008, | |
| "num_tokens": 936513.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/reward_func": 0.125, | |
| "step": 50 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 484.40625, | |
| "epoch": 0.10024570024570024, | |
| "grad_norm": 0.8466055231288873, | |
| "kl": 0.01735687255859375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0053, | |
| "num_tokens": 955662.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/reward_func": 0.125, | |
| "step": 51 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 603.40625, | |
| "epoch": 0.10221130221130222, | |
| "grad_norm": 0.4594988801570104, | |
| "kl": 0.01914215087890625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0157, | |
| "num_tokens": 978603.0, | |
| "reward": 0.0625, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/reward_func": 0.0625, | |
| "step": 52 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 571.53125, | |
| "epoch": 0.10417690417690417, | |
| "grad_norm": 0.6301487761037352, | |
| "kl": 0.01680755615234375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0518, | |
| "num_tokens": 1000414.0, | |
| "reward": 0.125, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/reward_func": 0.125, | |
| "step": 53 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 480.625, | |
| "epoch": 0.10614250614250614, | |
| "grad_norm": 0.1705061403375656, | |
| "kl": 0.019412994384765625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0151, | |
| "num_tokens": 1019984.0, | |
| "reward": 0.03125, | |
| "reward_std": 0.04419417306780815, | |
| "rewards/reward_func": 0.03125, | |
| "step": 54 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 586.5625, | |
| "epoch": 0.10810810810810811, | |
| "grad_norm": 0.3563070736118851, | |
| "kl": 0.01416778564453125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0113, | |
| "num_tokens": 1043134.0, | |
| "reward": 0.03125, | |
| "reward_std": 0.04419417306780815, | |
| "rewards/reward_func": 0.03125, | |
| "step": 55 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 573.78125, | |
| "epoch": 0.11007371007371007, | |
| "grad_norm": 0.8170380900607339, | |
| "kl": 0.0185394287109375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0107, | |
| "num_tokens": 1065473.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/reward_func": 0.125, | |
| "step": 56 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 596.25, | |
| "epoch": 0.11203931203931204, | |
| "grad_norm": 0.6138181155074318, | |
| "kl": 0.01763916015625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0173, | |
| "num_tokens": 1088233.0, | |
| "reward": 0.0625, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/reward_func": 0.0625, | |
| "step": 57 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 579.96875, | |
| "epoch": 0.114004914004914, | |
| "grad_norm": 0.5592170560306685, | |
| "kl": 0.015392303466796875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.042, | |
| "num_tokens": 1111490.0, | |
| "reward": 0.125, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/reward_func": 0.125, | |
| "step": 58 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 590.3125, | |
| "epoch": 0.11597051597051597, | |
| "grad_norm": 0.004534498568535779, | |
| "kl": 0.017578125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0, | |
| "num_tokens": 1134000.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0, | |
| "rewards/reward_func": 0.125, | |
| "step": 59 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 552.75, | |
| "epoch": 0.11793611793611794, | |
| "grad_norm": 0.26122008088795556, | |
| "kl": 0.019062042236328125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0, | |
| "num_tokens": 1155236.0, | |
| "reward": 0.09375, | |
| "reward_std": 0.04419417306780815, | |
| "rewards/reward_func": 0.09375, | |
| "step": 60 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 533.46875, | |
| "epoch": 0.1199017199017199, | |
| "grad_norm": 0.6246443042334695, | |
| "kl": 0.01920318603515625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0014, | |
| "num_tokens": 1175617.0, | |
| "reward": 0.09375, | |
| "reward_std": 0.13258251920342445, | |
| "rewards/reward_func": 0.09375, | |
| "step": 61 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 484.34375, | |
| "epoch": 0.12186732186732187, | |
| "grad_norm": 1.0553043786365885, | |
| "kl": 0.0269622802734375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0461, | |
| "num_tokens": 1194332.0, | |
| "reward": 0.21875, | |
| "reward_std": 0.22097086533904076, | |
| "rewards/reward_func": 0.21875, | |
| "step": 62 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 604.84375, | |
| "epoch": 0.12383292383292384, | |
| "grad_norm": 0.5614007392931322, | |
| "kl": 0.0162506103515625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0098, | |
| "num_tokens": 1217363.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/reward_func": 0.1875, | |
| "step": 63 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 579.59375, | |
| "epoch": 0.1257985257985258, | |
| "grad_norm": 0.001324981221462767, | |
| "kl": 0.0164794921875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0, | |
| "num_tokens": 1239884.0, | |
| "reward": 0.0625, | |
| "reward_std": 0.0, | |
| "rewards/reward_func": 0.0625, | |
| "step": 64 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 643.0, | |
| "epoch": 0.12776412776412777, | |
| "grad_norm": 0.6061011991202276, | |
| "kl": 0.01563262939453125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.065, | |
| "num_tokens": 1263644.0, | |
| "reward": 0.125, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/reward_func": 0.125, | |
| "step": 65 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 487.875, | |
| "epoch": 0.12972972972972974, | |
| "grad_norm": 0.8366609324695296, | |
| "kl": 0.02077484130859375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0478, | |
| "num_tokens": 1282820.0, | |
| "reward": 0.15625, | |
| "reward_std": 0.22097086533904076, | |
| "rewards/reward_func": 0.15625, | |
| "step": 66 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 662.59375, | |
| "epoch": 0.1316953316953317, | |
| "grad_norm": 0.17490281761649612, | |
| "kl": 0.0147705078125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0204, | |
| "num_tokens": 1307457.0, | |
| "reward": 0.09375, | |
| "reward_std": 0.04419417306780815, | |
| "rewards/reward_func": 0.09375, | |
| "step": 67 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 525.03125, | |
| "epoch": 0.13366093366093365, | |
| "grad_norm": 0.8215530702995869, | |
| "kl": 0.020721435546875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.064, | |
| "num_tokens": 1327798.0, | |
| "reward": 0.25, | |
| "reward_std": 0.3535533845424652, | |
| "rewards/reward_func": 0.25, | |
| "step": 68 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 567.1875, | |
| "epoch": 0.13562653562653562, | |
| "grad_norm": 0.65089823105939, | |
| "kl": 0.019927978515625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0026, | |
| "num_tokens": 1350334.0, | |
| "reward": 0.15625, | |
| "reward_std": 0.22097086533904076, | |
| "rewards/reward_func": 0.15625, | |
| "step": 69 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 584.78125, | |
| "epoch": 0.1375921375921376, | |
| "grad_norm": 1.008422919649498, | |
| "kl": 0.01912689208984375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0025, | |
| "num_tokens": 1372311.0, | |
| "reward": 0.25, | |
| "reward_std": 0.2651650384068489, | |
| "rewards/reward_func": 0.25, | |
| "step": 70 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 545.125, | |
| "epoch": 0.13955773955773956, | |
| "grad_norm": 0.6752475010590075, | |
| "kl": 0.01819610595703125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0177, | |
| "num_tokens": 1393645.0, | |
| "reward": 0.21875, | |
| "reward_std": 0.22097086533904076, | |
| "rewards/reward_func": 0.21875, | |
| "step": 71 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 526.40625, | |
| "epoch": 0.14152334152334153, | |
| "grad_norm": 0.7868187210048034, | |
| "kl": 0.0257568359375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0076, | |
| "num_tokens": 1414260.0, | |
| "reward": 0.15625, | |
| "reward_std": 0.13258251920342445, | |
| "rewards/reward_func": 0.15625, | |
| "step": 72 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 465.46875, | |
| "epoch": 0.14348894348894348, | |
| "grad_norm": 0.40424658361331584, | |
| "kl": 0.02496337890625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0034, | |
| "num_tokens": 1432617.0, | |
| "reward": 0.09375, | |
| "reward_std": 0.04419417306780815, | |
| "rewards/reward_func": 0.09375, | |
| "step": 73 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 523.0, | |
| "epoch": 0.14545454545454545, | |
| "grad_norm": 0.7217516854340651, | |
| "kl": 0.0191497802734375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0438, | |
| "num_tokens": 1452631.0, | |
| "reward": 0.125, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/reward_func": 0.125, | |
| "step": 74 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 564.59375, | |
| "epoch": 0.14742014742014742, | |
| "grad_norm": 1.020886020177138, | |
| "kl": 0.02172088623046875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0443, | |
| "num_tokens": 1474180.0, | |
| "reward": 0.25, | |
| "reward_std": 0.2651650384068489, | |
| "rewards/reward_func": 0.25, | |
| "step": 75 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 599.03125, | |
| "epoch": 0.1493857493857494, | |
| "grad_norm": 0.38779629766219476, | |
| "kl": 0.021942138671875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0253, | |
| "num_tokens": 1496597.0, | |
| "reward": 0.0625, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/reward_func": 0.0625, | |
| "step": 76 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 535.65625, | |
| "epoch": 0.15135135135135136, | |
| "grad_norm": 0.5231239100329028, | |
| "kl": 0.0213470458984375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0072, | |
| "num_tokens": 1518002.0, | |
| "reward": 0.09375, | |
| "reward_std": 0.13258251920342445, | |
| "rewards/reward_func": 0.09375, | |
| "step": 77 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 483.28125, | |
| "epoch": 0.15331695331695333, | |
| "grad_norm": 1.01964836669152, | |
| "kl": 0.02301025390625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0253, | |
| "num_tokens": 1538251.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.2651650384068489, | |
| "rewards/reward_func": 0.1875, | |
| "step": 78 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 597.21875, | |
| "epoch": 0.15528255528255527, | |
| "grad_norm": 0.3894010167090417, | |
| "kl": 0.01868438720703125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0049, | |
| "num_tokens": 1561066.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/reward_func": 0.125, | |
| "step": 79 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 494.21875, | |
| "epoch": 0.15724815724815724, | |
| "grad_norm": 0.4559857762776204, | |
| "kl": 0.02178955078125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0144, | |
| "num_tokens": 1579851.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/reward_func": 0.125, | |
| "step": 80 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 459.4375, | |
| "epoch": 0.15921375921375922, | |
| "grad_norm": 0.7408328801166559, | |
| "kl": 0.02642822265625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0034, | |
| "num_tokens": 1597675.0, | |
| "reward": 0.15625, | |
| "reward_std": 0.13258251920342445, | |
| "rewards/reward_func": 0.15625, | |
| "step": 81 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 601.90625, | |
| "epoch": 0.16117936117936119, | |
| "grad_norm": 0.4202464041477536, | |
| "kl": 0.01740264892578125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0084, | |
| "num_tokens": 1621632.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/reward_func": 0.125, | |
| "step": 82 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 517.21875, | |
| "epoch": 0.16314496314496316, | |
| "grad_norm": 0.7785902853786559, | |
| "kl": 0.02386474609375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0036, | |
| "num_tokens": 1641873.0, | |
| "reward": 0.21875, | |
| "reward_std": 0.22097086533904076, | |
| "rewards/reward_func": 0.21875, | |
| "step": 83 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 390.6875, | |
| "epoch": 0.1651105651105651, | |
| "grad_norm": 0.6791841008394204, | |
| "kl": 0.034637451171875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0184, | |
| "num_tokens": 1658453.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/reward_func": 0.1875, | |
| "step": 84 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 485.78125, | |
| "epoch": 0.16707616707616707, | |
| "grad_norm": 0.4367719984766766, | |
| "kl": 0.0251617431640625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0156, | |
| "num_tokens": 1676992.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/reward_func": 0.125, | |
| "step": 85 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 528.875, | |
| "epoch": 0.16904176904176904, | |
| "grad_norm": 0.7222029349504244, | |
| "kl": 0.02223968505859375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0459, | |
| "num_tokens": 1700584.0, | |
| "reward": 0.21875, | |
| "reward_std": 0.22097086533904076, | |
| "rewards/reward_func": 0.21875, | |
| "step": 86 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 658.125, | |
| "epoch": 0.171007371007371, | |
| "grad_norm": 0.34834776108087606, | |
| "kl": 0.01824951171875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0326, | |
| "num_tokens": 1725524.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/reward_func": 0.125, | |
| "step": 87 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 644.5625, | |
| "epoch": 0.17297297297297298, | |
| "grad_norm": 0.6750518830704131, | |
| "kl": 0.01828765869140625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0412, | |
| "num_tokens": 1749556.0, | |
| "reward": 0.15625, | |
| "reward_std": 0.22097086533904076, | |
| "rewards/reward_func": 0.15625, | |
| "step": 88 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 547.15625, | |
| "epoch": 0.17493857493857493, | |
| "grad_norm": 0.31981853904702845, | |
| "kl": 0.01940155029296875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.012, | |
| "num_tokens": 1770175.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/reward_func": 0.1875, | |
| "step": 89 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 475.75, | |
| "epoch": 0.1769041769041769, | |
| "grad_norm": 1.176119812297408, | |
| "kl": 0.0265045166015625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0174, | |
| "num_tokens": 1789467.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.2651650384068489, | |
| "rewards/reward_func": 0.1875, | |
| "step": 90 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 448.625, | |
| "epoch": 0.17886977886977887, | |
| "grad_norm": 0.7247013137515755, | |
| "kl": 0.02973175048828125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0025, | |
| "num_tokens": 1807295.0, | |
| "reward": 0.09375, | |
| "reward_std": 0.13258251920342445, | |
| "rewards/reward_func": 0.09375, | |
| "step": 91 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 461.84375, | |
| "epoch": 0.18083538083538084, | |
| "grad_norm": 0.9576563539486035, | |
| "kl": 0.027984619140625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0572, | |
| "num_tokens": 1827104.0, | |
| "reward": 0.28125, | |
| "reward_std": 0.22097086533904076, | |
| "rewards/reward_func": 0.28125, | |
| "step": 92 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 549.375, | |
| "epoch": 0.1828009828009828, | |
| "grad_norm": 0.8080609769164848, | |
| "kl": 0.02375030517578125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0641, | |
| "num_tokens": 1848272.0, | |
| "reward": 0.21875, | |
| "reward_std": 0.22097086533904076, | |
| "rewards/reward_func": 0.21875, | |
| "step": 93 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 673.53125, | |
| "epoch": 0.18476658476658478, | |
| "grad_norm": 0.6070813087390035, | |
| "kl": 0.0193634033203125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0232, | |
| "num_tokens": 1873533.0, | |
| "reward": 0.125, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/reward_func": 0.125, | |
| "step": 94 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 582.75, | |
| "epoch": 0.18673218673218672, | |
| "grad_norm": 0.6479257231531951, | |
| "kl": 0.0185394287109375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0017, | |
| "num_tokens": 1895601.0, | |
| "reward": 0.3125, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/reward_func": 0.3125, | |
| "step": 95 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 443.5, | |
| "epoch": 0.1886977886977887, | |
| "grad_norm": 0.7375270346991556, | |
| "kl": 0.0291290283203125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0343, | |
| "num_tokens": 1912993.0, | |
| "reward": 0.15625, | |
| "reward_std": 0.13258251920342445, | |
| "rewards/reward_func": 0.15625, | |
| "step": 96 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 499.125, | |
| "epoch": 0.19066339066339066, | |
| "grad_norm": 0.6426005454499798, | |
| "kl": 0.0281524658203125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0165, | |
| "num_tokens": 1933631.0, | |
| "reward": 0.21875, | |
| "reward_std": 0.22097086533904076, | |
| "rewards/reward_func": 0.21875, | |
| "step": 97 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 583.21875, | |
| "epoch": 0.19262899262899263, | |
| "grad_norm": 0.656305823795339, | |
| "kl": 0.0197601318359375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0098, | |
| "num_tokens": 1955588.0, | |
| "reward": 0.125, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/reward_func": 0.125, | |
| "step": 98 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 604.5, | |
| "epoch": 0.1945945945945946, | |
| "grad_norm": 0.7159925904721693, | |
| "kl": 0.0202484130859375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0024, | |
| "num_tokens": 1978466.0, | |
| "reward": 0.15625, | |
| "reward_std": 0.22097086533904076, | |
| "rewards/reward_func": 0.15625, | |
| "step": 99 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 623.8125, | |
| "epoch": 0.19656019656019655, | |
| "grad_norm": 0.6011532165345448, | |
| "kl": 0.0197296142578125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.015, | |
| "num_tokens": 2001554.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/reward_func": 0.1875, | |
| "step": 100 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 487.84375, | |
| "epoch": 0.19852579852579852, | |
| "grad_norm": 0.6696267166180018, | |
| "kl": 0.02446746826171875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0323, | |
| "num_tokens": 2020681.0, | |
| "reward": 0.21875, | |
| "reward_std": 0.22097086533904076, | |
| "rewards/reward_func": 0.21875, | |
| "step": 101 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 512.90625, | |
| "epoch": 0.2004914004914005, | |
| "grad_norm": 0.5144645053944626, | |
| "kl": 0.0222320556640625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0308, | |
| "num_tokens": 2040058.0, | |
| "reward": 0.0625, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/reward_func": 0.0625, | |
| "step": 102 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 492.59375, | |
| "epoch": 0.20245700245700246, | |
| "grad_norm": 0.9792203746023931, | |
| "kl": 0.02777099609375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0185, | |
| "num_tokens": 2059481.0, | |
| "reward": 0.28125, | |
| "reward_std": 0.22097086533904076, | |
| "rewards/reward_func": 0.28125, | |
| "step": 103 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 540.5625, | |
| "epoch": 0.20442260442260443, | |
| "grad_norm": 0.6773524274596018, | |
| "kl": 0.0245513916015625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0115, | |
| "num_tokens": 2079795.0, | |
| "reward": 0.125, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/reward_func": 0.125, | |
| "step": 104 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 629.75, | |
| "epoch": 0.20638820638820637, | |
| "grad_norm": 0.0014408090990997875, | |
| "kl": 0.020843505859375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0, | |
| "num_tokens": 2104281.0, | |
| "reward": 0.0, | |
| "reward_std": 0.0, | |
| "rewards/reward_func": 0.0, | |
| "step": 105 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 489.625, | |
| "epoch": 0.20835380835380835, | |
| "grad_norm": 0.43577632912535613, | |
| "kl": 0.02980804443359375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0265, | |
| "num_tokens": 2124275.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/reward_func": 0.25, | |
| "step": 106 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 480.90625, | |
| "epoch": 0.21031941031941032, | |
| "grad_norm": 0.5136803725458018, | |
| "kl": 0.02611541748046875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0184, | |
| "num_tokens": 2143080.0, | |
| "reward": 0.3125, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/reward_func": 0.3125, | |
| "step": 107 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 659.9375, | |
| "epoch": 0.2122850122850123, | |
| "grad_norm": 0.4897037723304078, | |
| "kl": 0.0196380615234375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0062, | |
| "num_tokens": 2167550.0, | |
| "reward": 0.15625, | |
| "reward_std": 0.13258251920342445, | |
| "rewards/reward_func": 0.15625, | |
| "step": 108 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 549.53125, | |
| "epoch": 0.21425061425061426, | |
| "grad_norm": 0.6823729897684875, | |
| "kl": 0.020660400390625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0027, | |
| "num_tokens": 2188815.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.2651650384068489, | |
| "rewards/reward_func": 0.1875, | |
| "step": 109 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 492.4375, | |
| "epoch": 0.21621621621621623, | |
| "grad_norm": 0.8520053469926316, | |
| "kl": 0.02719879150390625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0077, | |
| "num_tokens": 2207977.0, | |
| "reward": 0.28125, | |
| "reward_std": 0.22097086533904076, | |
| "rewards/reward_func": 0.28125, | |
| "step": 110 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 487.34375, | |
| "epoch": 0.21818181818181817, | |
| "grad_norm": 0.8983378201666078, | |
| "kl": 0.024871826171875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0207, | |
| "num_tokens": 2226818.0, | |
| "reward": 0.40625, | |
| "reward_std": 0.22097086533904076, | |
| "rewards/reward_func": 0.40625, | |
| "step": 111 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 534.71875, | |
| "epoch": 0.22014742014742014, | |
| "grad_norm": 0.6341326642846564, | |
| "kl": 0.022857666015625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.005, | |
| "num_tokens": 2247351.0, | |
| "reward": 0.25, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/reward_func": 0.25, | |
| "step": 112 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 509.96875, | |
| "epoch": 0.2221130221130221, | |
| "grad_norm": 0.5084014496191064, | |
| "kl": 0.02394866943359375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0121, | |
| "num_tokens": 2267646.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/reward_func": 0.1875, | |
| "step": 113 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 493.46875, | |
| "epoch": 0.22407862407862408, | |
| "grad_norm": 0.47081172119878306, | |
| "kl": 0.023681640625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.003, | |
| "num_tokens": 2287461.0, | |
| "reward": 0.21875, | |
| "reward_std": 0.13258251920342445, | |
| "rewards/reward_func": 0.21875, | |
| "step": 114 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 593.46875, | |
| "epoch": 0.22604422604422605, | |
| "grad_norm": 0.5402745390200868, | |
| "kl": 0.0226593017578125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0254, | |
| "num_tokens": 2310300.0, | |
| "reward": 0.21875, | |
| "reward_std": 0.13258251920342445, | |
| "rewards/reward_func": 0.21875, | |
| "step": 115 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 659.34375, | |
| "epoch": 0.228009828009828, | |
| "grad_norm": 0.7384552819583179, | |
| "kl": 0.01863861083984375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.011, | |
| "num_tokens": 2336289.0, | |
| "reward": 0.21875, | |
| "reward_std": 0.22097086533904076, | |
| "rewards/reward_func": 0.21875, | |
| "step": 116 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 426.0625, | |
| "epoch": 0.22997542997542997, | |
| "grad_norm": 0.9856793371703316, | |
| "kl": 0.029998779296875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0107, | |
| "num_tokens": 2353069.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.2651650384068489, | |
| "rewards/reward_func": 0.1875, | |
| "step": 117 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 553.15625, | |
| "epoch": 0.23194103194103194, | |
| "grad_norm": 0.6524109632549928, | |
| "kl": 0.0211334228515625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0166, | |
| "num_tokens": 2374500.0, | |
| "reward": 0.25, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/reward_func": 0.25, | |
| "step": 118 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 495.96875, | |
| "epoch": 0.2339066339066339, | |
| "grad_norm": 0.750164710998597, | |
| "kl": 0.0268707275390625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0291, | |
| "num_tokens": 2393455.0, | |
| "reward": 0.34375, | |
| "reward_std": 0.13258251920342445, | |
| "rewards/reward_func": 0.34375, | |
| "step": 119 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 519.375, | |
| "epoch": 0.23587223587223588, | |
| "grad_norm": 0.8136338780518924, | |
| "kl": 0.0233917236328125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0213, | |
| "num_tokens": 2413821.0, | |
| "reward": 0.25, | |
| "reward_std": 0.2651650384068489, | |
| "rewards/reward_func": 0.25, | |
| "step": 120 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 492.5, | |
| "epoch": 0.23783783783783785, | |
| "grad_norm": 0.9472454617054751, | |
| "kl": 0.0245513916015625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0434, | |
| "num_tokens": 2432727.0, | |
| "reward": 0.3125, | |
| "reward_std": 0.3535533845424652, | |
| "rewards/reward_func": 0.3125, | |
| "step": 121 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 650.125, | |
| "epoch": 0.2398034398034398, | |
| "grad_norm": 0.7941644508393335, | |
| "kl": 0.0188446044921875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0161, | |
| "num_tokens": 2457059.0, | |
| "reward": 0.21875, | |
| "reward_std": 0.22097086533904076, | |
| "rewards/reward_func": 0.21875, | |
| "step": 122 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 505.40625, | |
| "epoch": 0.24176904176904176, | |
| "grad_norm": 0.9756007077675318, | |
| "kl": 0.02396392822265625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0303, | |
| "num_tokens": 2477004.0, | |
| "reward": 0.53125, | |
| "reward_std": 0.30935921147465706, | |
| "rewards/reward_func": 0.53125, | |
| "step": 123 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 483.96875, | |
| "epoch": 0.24373464373464374, | |
| "grad_norm": 0.6941208484996456, | |
| "kl": 0.027313232421875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0341, | |
| "num_tokens": 2495959.0, | |
| "reward": 0.40625, | |
| "reward_std": 0.22097086533904076, | |
| "rewards/reward_func": 0.40625, | |
| "step": 124 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 561.3125, | |
| "epoch": 0.2457002457002457, | |
| "grad_norm": 0.5265733548592475, | |
| "kl": 0.01922607421875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0423, | |
| "num_tokens": 2517637.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/reward_func": 0.1875, | |
| "step": 125 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 492.96875, | |
| "epoch": 0.24766584766584768, | |
| "grad_norm": 0.8350574297158757, | |
| "kl": 0.023712158203125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0469, | |
| "num_tokens": 2538338.0, | |
| "reward": 0.34375, | |
| "reward_std": 0.22097086533904076, | |
| "rewards/reward_func": 0.34375, | |
| "step": 126 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 573.5, | |
| "epoch": 0.24963144963144962, | |
| "grad_norm": 0.8216738883653201, | |
| "kl": 0.02204132080078125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0218, | |
| "num_tokens": 2560112.0, | |
| "reward": 0.28125, | |
| "reward_std": 0.30935921147465706, | |
| "rewards/reward_func": 0.28125, | |
| "step": 127 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 529.3125, | |
| "epoch": 0.2515970515970516, | |
| "grad_norm": 0.78248285660239, | |
| "kl": 0.02326202392578125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.044, | |
| "num_tokens": 2580546.0, | |
| "reward": 0.34375, | |
| "reward_std": 0.22097086533904076, | |
| "rewards/reward_func": 0.34375, | |
| "step": 128 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 602.1875, | |
| "epoch": 0.25356265356265356, | |
| "grad_norm": 0.2772418236153871, | |
| "kl": 0.019317626953125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0027, | |
| "num_tokens": 2604460.0, | |
| "reward": 0.15625, | |
| "reward_std": 0.04419417306780815, | |
| "rewards/reward_func": 0.15625, | |
| "step": 129 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 548.0625, | |
| "epoch": 0.25552825552825553, | |
| "grad_norm": 0.6625525706905956, | |
| "kl": 0.02095794677734375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0115, | |
| "num_tokens": 2624972.0, | |
| "reward": 0.21875, | |
| "reward_std": 0.22097086533904076, | |
| "rewards/reward_func": 0.21875, | |
| "step": 130 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 585.9375, | |
| "epoch": 0.2574938574938575, | |
| "grad_norm": 0.752169210277677, | |
| "kl": 0.021514892578125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0359, | |
| "num_tokens": 2648150.0, | |
| "reward": 0.375, | |
| "reward_std": 0.2651650384068489, | |
| "rewards/reward_func": 0.375, | |
| "step": 131 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 574.0625, | |
| "epoch": 0.2594594594594595, | |
| "grad_norm": 0.5002363204112915, | |
| "kl": 0.0212249755859375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.023, | |
| "num_tokens": 2670338.0, | |
| "reward": 0.28125, | |
| "reward_std": 0.13258251920342445, | |
| "rewards/reward_func": 0.28125, | |
| "step": 132 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 526.625, | |
| "epoch": 0.26142506142506144, | |
| "grad_norm": 0.9674758127355131, | |
| "kl": 0.0261077880859375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0159, | |
| "num_tokens": 2691642.0, | |
| "reward": 0.34375, | |
| "reward_std": 0.30935921147465706, | |
| "rewards/reward_func": 0.34375, | |
| "step": 133 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 412.59375, | |
| "epoch": 0.2633906633906634, | |
| "grad_norm": 1.0324181511597994, | |
| "kl": 0.0365447998046875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0109, | |
| "num_tokens": 2708471.0, | |
| "reward": 0.53125, | |
| "reward_std": 0.30935921147465706, | |
| "rewards/reward_func": 0.53125, | |
| "step": 134 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 501.1875, | |
| "epoch": 0.26535626535626533, | |
| "grad_norm": 0.646085784140391, | |
| "kl": 0.02829742431640625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0245, | |
| "num_tokens": 2728631.0, | |
| "reward": 0.34375, | |
| "reward_std": 0.13258251920342445, | |
| "rewards/reward_func": 0.34375, | |
| "step": 135 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 524.59375, | |
| "epoch": 0.2673218673218673, | |
| "grad_norm": 0.5827914647557386, | |
| "kl": 0.0233001708984375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0133, | |
| "num_tokens": 2748682.0, | |
| "reward": 0.375, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/reward_func": 0.375, | |
| "step": 136 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 493.625, | |
| "epoch": 0.26928746928746927, | |
| "grad_norm": 0.5383283228577356, | |
| "kl": 0.02911376953125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.006, | |
| "num_tokens": 2768384.0, | |
| "reward": 0.15625, | |
| "reward_std": 0.04419417306780815, | |
| "rewards/reward_func": 0.15625, | |
| "step": 137 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 563.34375, | |
| "epoch": 0.27125307125307124, | |
| "grad_norm": 0.44514691751088103, | |
| "kl": 0.02154541015625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.045, | |
| "num_tokens": 2790263.0, | |
| "reward": 0.15625, | |
| "reward_std": 0.13258251920342445, | |
| "rewards/reward_func": 0.15625, | |
| "step": 138 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 520.8125, | |
| "epoch": 0.2732186732186732, | |
| "grad_norm": 0.7190292298516152, | |
| "kl": 0.0236663818359375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0132, | |
| "num_tokens": 2810351.0, | |
| "reward": 0.375, | |
| "reward_std": 0.2651650384068489, | |
| "rewards/reward_func": 0.375, | |
| "step": 139 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 527.34375, | |
| "epoch": 0.2751842751842752, | |
| "grad_norm": 0.7242965068485637, | |
| "kl": 0.0230255126953125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.001, | |
| "num_tokens": 2830716.0, | |
| "reward": 0.34375, | |
| "reward_std": 0.22097086533904076, | |
| "rewards/reward_func": 0.34375, | |
| "step": 140 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 565.8125, | |
| "epoch": 0.27714987714987716, | |
| "grad_norm": 0.6228089750555335, | |
| "kl": 0.02398681640625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0041, | |
| "num_tokens": 2853810.0, | |
| "reward": 0.34375, | |
| "reward_std": 0.13258251920342445, | |
| "rewards/reward_func": 0.34375, | |
| "step": 141 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 555.1875, | |
| "epoch": 0.2791154791154791, | |
| "grad_norm": 0.37035042072698693, | |
| "kl": 0.02410888671875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0047, | |
| "num_tokens": 2875056.0, | |
| "reward": 0.3125, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/reward_func": 0.3125, | |
| "step": 142 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 589.65625, | |
| "epoch": 0.2810810810810811, | |
| "grad_norm": 0.658035150283911, | |
| "kl": 0.02394866943359375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.025, | |
| "num_tokens": 2897357.0, | |
| "reward": 0.4375, | |
| "reward_std": 0.2651650384068489, | |
| "rewards/reward_func": 0.4375, | |
| "step": 143 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 543.8125, | |
| "epoch": 0.28304668304668307, | |
| "grad_norm": 0.7501331474001123, | |
| "kl": 0.025238037109375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0123, | |
| "num_tokens": 2917753.0, | |
| "reward": 0.25, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/reward_func": 0.25, | |
| "step": 144 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 612.28125, | |
| "epoch": 0.28501228501228504, | |
| "grad_norm": 0.5446469883013957, | |
| "kl": 0.0233612060546875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0054, | |
| "num_tokens": 2940774.0, | |
| "reward": 0.3125, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/reward_func": 0.3125, | |
| "step": 145 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 539.78125, | |
| "epoch": 0.28697788697788695, | |
| "grad_norm": 0.6092756185830317, | |
| "kl": 0.02280426025390625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0263, | |
| "num_tokens": 2961247.0, | |
| "reward": 0.34375, | |
| "reward_std": 0.13258251920342445, | |
| "rewards/reward_func": 0.34375, | |
| "step": 146 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 466.28125, | |
| "epoch": 0.2889434889434889, | |
| "grad_norm": 0.6941889736363939, | |
| "kl": 0.0313873291015625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0046, | |
| "num_tokens": 2979622.0, | |
| "reward": 0.09375, | |
| "reward_std": 0.13258251920342445, | |
| "rewards/reward_func": 0.09375, | |
| "step": 147 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 567.53125, | |
| "epoch": 0.2909090909090909, | |
| "grad_norm": 0.3736888500181184, | |
| "kl": 0.023223876953125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0099, | |
| "num_tokens": 3002039.0, | |
| "reward": 0.15625, | |
| "reward_std": 0.13258251920342445, | |
| "rewards/reward_func": 0.15625, | |
| "step": 148 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 467.59375, | |
| "epoch": 0.29287469287469287, | |
| "grad_norm": 0.8319282300423898, | |
| "kl": 0.02935791015625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.043, | |
| "num_tokens": 3019816.0, | |
| "reward": 0.375, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/reward_func": 0.375, | |
| "step": 149 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 492.25, | |
| "epoch": 0.29484029484029484, | |
| "grad_norm": 0.7325276211486558, | |
| "kl": 0.0263671875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0293, | |
| "num_tokens": 3039146.0, | |
| "reward": 0.46875, | |
| "reward_std": 0.22097086533904076, | |
| "rewards/reward_func": 0.46875, | |
| "step": 150 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 592.1875, | |
| "epoch": 0.2968058968058968, | |
| "grad_norm": 0.43633918201867017, | |
| "kl": 0.0228424072265625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0122, | |
| "num_tokens": 3062102.0, | |
| "reward": 0.34375, | |
| "reward_std": 0.13258251920342445, | |
| "rewards/reward_func": 0.34375, | |
| "step": 151 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 563.1875, | |
| "epoch": 0.2987714987714988, | |
| "grad_norm": 0.38992921040393713, | |
| "kl": 0.025421142578125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0327, | |
| "num_tokens": 3084284.0, | |
| "reward": 0.28125, | |
| "reward_std": 0.13258251920342445, | |
| "rewards/reward_func": 0.28125, | |
| "step": 152 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 634.9375, | |
| "epoch": 0.30073710073710075, | |
| "grad_norm": 0.41943188112426816, | |
| "kl": 0.0197906494140625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.001, | |
| "num_tokens": 3108216.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/reward_func": 0.1875, | |
| "step": 153 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 636.9375, | |
| "epoch": 0.3027027027027027, | |
| "grad_norm": 0.19968498604663434, | |
| "kl": 0.02194976806640625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0001, | |
| "num_tokens": 3133100.0, | |
| "reward": 0.09375, | |
| "reward_std": 0.04419417306780815, | |
| "rewards/reward_func": 0.09375, | |
| "step": 154 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 568.21875, | |
| "epoch": 0.3046683046683047, | |
| "grad_norm": 0.7545074739064522, | |
| "kl": 0.02685546875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.04, | |
| "num_tokens": 3154579.0, | |
| "reward": 0.46875, | |
| "reward_std": 0.22097086533904076, | |
| "rewards/reward_func": 0.46875, | |
| "step": 155 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 529.8125, | |
| "epoch": 0.30663390663390666, | |
| "grad_norm": 0.6884386708248261, | |
| "kl": 0.025360107421875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0075, | |
| "num_tokens": 3174845.0, | |
| "reward": 0.40625, | |
| "reward_std": 0.22097086533904076, | |
| "rewards/reward_func": 0.40625, | |
| "step": 156 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 513.0625, | |
| "epoch": 0.3085995085995086, | |
| "grad_norm": 0.5265944726858988, | |
| "kl": 0.0264892578125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0245, | |
| "num_tokens": 3195631.0, | |
| "reward": 0.21875, | |
| "reward_std": 0.13258251920342445, | |
| "rewards/reward_func": 0.21875, | |
| "step": 157 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 520.0, | |
| "epoch": 0.31056511056511055, | |
| "grad_norm": 0.45924555811174794, | |
| "kl": 0.026092529296875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0189, | |
| "num_tokens": 3216143.0, | |
| "reward": 0.21875, | |
| "reward_std": 0.13258251920342445, | |
| "rewards/reward_func": 0.21875, | |
| "step": 158 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 568.15625, | |
| "epoch": 0.3125307125307125, | |
| "grad_norm": 0.5301940769841443, | |
| "kl": 0.025177001953125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0008, | |
| "num_tokens": 3238264.0, | |
| "reward": 0.15625, | |
| "reward_std": 0.13258251920342445, | |
| "rewards/reward_func": 0.15625, | |
| "step": 159 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 508.59375, | |
| "epoch": 0.3144963144963145, | |
| "grad_norm": 0.8755698562076372, | |
| "kl": 0.0290069580078125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0536, | |
| "num_tokens": 3257837.0, | |
| "reward": 0.34375, | |
| "reward_std": 0.22097086533904076, | |
| "rewards/reward_func": 0.34375, | |
| "step": 160 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 481.15625, | |
| "epoch": 0.31646191646191646, | |
| "grad_norm": 0.6081291396581243, | |
| "kl": 0.0274810791015625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0018, | |
| "num_tokens": 3276820.0, | |
| "reward": 0.4375, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/reward_func": 0.4375, | |
| "step": 161 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 598.28125, | |
| "epoch": 0.31842751842751843, | |
| "grad_norm": 0.5215378127068343, | |
| "kl": 0.0220794677734375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0012, | |
| "num_tokens": 3299357.0, | |
| "reward": 0.375, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/reward_func": 0.375, | |
| "step": 162 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 487.09375, | |
| "epoch": 0.3203931203931204, | |
| "grad_norm": 0.5432850798950747, | |
| "kl": 0.03083038330078125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0034, | |
| "num_tokens": 3318084.0, | |
| "reward": 0.5, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/reward_func": 0.5, | |
| "step": 163 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 531.125, | |
| "epoch": 0.32235872235872237, | |
| "grad_norm": 0.46027668679847555, | |
| "kl": 0.026092529296875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0154, | |
| "num_tokens": 3338328.0, | |
| "reward": 0.125, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/reward_func": 0.125, | |
| "step": 164 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 624.8125, | |
| "epoch": 0.32432432432432434, | |
| "grad_norm": 0.18474075838750667, | |
| "kl": 0.02459716796875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0011, | |
| "num_tokens": 3361788.0, | |
| "reward": 0.21875, | |
| "reward_std": 0.04419417306780815, | |
| "rewards/reward_func": 0.21875, | |
| "step": 165 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 549.78125, | |
| "epoch": 0.3262899262899263, | |
| "grad_norm": 0.7718886882448309, | |
| "kl": 0.027069091796875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.005, | |
| "num_tokens": 3382981.0, | |
| "reward": 0.46875, | |
| "reward_std": 0.30935921147465706, | |
| "rewards/reward_func": 0.46875, | |
| "step": 166 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 575.78125, | |
| "epoch": 0.32825552825552823, | |
| "grad_norm": 0.5895346584970902, | |
| "kl": 0.02323150634765625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0178, | |
| "num_tokens": 3405274.0, | |
| "reward": 0.21875, | |
| "reward_std": 0.22097086533904076, | |
| "rewards/reward_func": 0.21875, | |
| "step": 167 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 576.875, | |
| "epoch": 0.3302211302211302, | |
| "grad_norm": 0.4959807619539926, | |
| "kl": 0.0241851806640625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0022, | |
| "num_tokens": 3427392.0, | |
| "reward": 0.21875, | |
| "reward_std": 0.13258251920342445, | |
| "rewards/reward_func": 0.21875, | |
| "step": 168 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 561.78125, | |
| "epoch": 0.33218673218673217, | |
| "grad_norm": 0.8681759456851222, | |
| "kl": 0.0249176025390625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0274, | |
| "num_tokens": 3448681.0, | |
| "reward": 0.34375, | |
| "reward_std": 0.30935921147465706, | |
| "rewards/reward_func": 0.34375, | |
| "step": 169 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 483.53125, | |
| "epoch": 0.33415233415233414, | |
| "grad_norm": 0.5297566985301704, | |
| "kl": 0.0336761474609375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0019, | |
| "num_tokens": 3467212.0, | |
| "reward": 0.21875, | |
| "reward_std": 0.13258251920342445, | |
| "rewards/reward_func": 0.21875, | |
| "step": 170 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 581.40625, | |
| "epoch": 0.3361179361179361, | |
| "grad_norm": 0.6192719333892511, | |
| "kl": 0.025787353515625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0075, | |
| "num_tokens": 3489355.0, | |
| "reward": 0.25, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/reward_func": 0.25, | |
| "step": 171 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 709.09375, | |
| "epoch": 0.3380835380835381, | |
| "grad_norm": 0.47423370876978443, | |
| "kl": 0.0204620361328125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0251, | |
| "num_tokens": 3515834.0, | |
| "reward": 0.21875, | |
| "reward_std": 0.13258251920342445, | |
| "rewards/reward_func": 0.21875, | |
| "step": 172 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 614.21875, | |
| "epoch": 0.34004914004914005, | |
| "grad_norm": 0.6479887044288859, | |
| "kl": 0.0229034423828125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0212, | |
| "num_tokens": 3539067.0, | |
| "reward": 0.3125, | |
| "reward_std": 0.2651650384068489, | |
| "rewards/reward_func": 0.3125, | |
| "step": 173 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 552.6875, | |
| "epoch": 0.342014742014742, | |
| "grad_norm": 0.4712074892004097, | |
| "kl": 0.0272216796875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0209, | |
| "num_tokens": 3560435.0, | |
| "reward": 0.21875, | |
| "reward_std": 0.13258251920342445, | |
| "rewards/reward_func": 0.21875, | |
| "step": 174 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 599.75, | |
| "epoch": 0.343980343980344, | |
| "grad_norm": 0.7325904251977917, | |
| "kl": 0.0259552001953125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0087, | |
| "num_tokens": 3582617.0, | |
| "reward": 0.25, | |
| "reward_std": 0.3535533845424652, | |
| "rewards/reward_func": 0.25, | |
| "step": 175 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 589.46875, | |
| "epoch": 0.34594594594594597, | |
| "grad_norm": 0.718404368627785, | |
| "kl": 0.024017333984375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0069, | |
| "num_tokens": 3606486.0, | |
| "reward": 0.21875, | |
| "reward_std": 0.22097086533904076, | |
| "rewards/reward_func": 0.21875, | |
| "step": 176 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 525.0625, | |
| "epoch": 0.34791154791154794, | |
| "grad_norm": 0.5807447821668908, | |
| "kl": 0.0262451171875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.006, | |
| "num_tokens": 3626528.0, | |
| "reward": 0.3125, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/reward_func": 0.3125, | |
| "step": 177 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 539.875, | |
| "epoch": 0.34987714987714985, | |
| "grad_norm": 0.3836295461022733, | |
| "kl": 0.025665283203125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0233, | |
| "num_tokens": 3647890.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/reward_func": 0.25, | |
| "step": 178 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 641.28125, | |
| "epoch": 0.3518427518427518, | |
| "grad_norm": 0.3325268331032711, | |
| "kl": 0.02361297607421875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0117, | |
| "num_tokens": 3672977.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/reward_func": 0.1875, | |
| "step": 179 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 611.1875, | |
| "epoch": 0.3538083538083538, | |
| "grad_norm": 0.5612204711943589, | |
| "kl": 0.025909423828125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.014, | |
| "num_tokens": 3696201.0, | |
| "reward": 0.28125, | |
| "reward_std": 0.22097086533904076, | |
| "rewards/reward_func": 0.28125, | |
| "step": 180 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 587.125, | |
| "epoch": 0.35577395577395576, | |
| "grad_norm": 0.48905557864261767, | |
| "kl": 0.0284881591796875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0135, | |
| "num_tokens": 3718645.0, | |
| "reward": 0.21875, | |
| "reward_std": 0.13258251920342445, | |
| "rewards/reward_func": 0.21875, | |
| "step": 181 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 532.8125, | |
| "epoch": 0.35773955773955773, | |
| "grad_norm": 0.6196793685491477, | |
| "kl": 0.0283050537109375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.021, | |
| "num_tokens": 3739419.0, | |
| "reward": 0.34375, | |
| "reward_std": 0.22097086533904076, | |
| "rewards/reward_func": 0.34375, | |
| "step": 182 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 491.25, | |
| "epoch": 0.3597051597051597, | |
| "grad_norm": 0.7943049071356907, | |
| "kl": 0.0270843505859375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0343, | |
| "num_tokens": 3758175.0, | |
| "reward": 0.3125, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/reward_func": 0.3125, | |
| "step": 183 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 629.3125, | |
| "epoch": 0.3616707616707617, | |
| "grad_norm": 0.4358875914582024, | |
| "kl": 0.02367401123046875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0133, | |
| "num_tokens": 3782789.0, | |
| "reward": 0.21875, | |
| "reward_std": 0.13258251920342445, | |
| "rewards/reward_func": 0.21875, | |
| "step": 184 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 540.59375, | |
| "epoch": 0.36363636363636365, | |
| "grad_norm": 0.7097269605040931, | |
| "kl": 0.028656005859375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.023, | |
| "num_tokens": 3804202.0, | |
| "reward": 0.25, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/reward_func": 0.25, | |
| "step": 185 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 517.40625, | |
| "epoch": 0.3656019656019656, | |
| "grad_norm": 0.8226505283368968, | |
| "kl": 0.0278778076171875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0374, | |
| "num_tokens": 3824813.0, | |
| "reward": 0.3125, | |
| "reward_std": 0.3535533845424652, | |
| "rewards/reward_func": 0.3125, | |
| "step": 186 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 499.3125, | |
| "epoch": 0.3675675675675676, | |
| "grad_norm": 0.8593511870826509, | |
| "kl": 0.026092529296875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0443, | |
| "num_tokens": 3844419.0, | |
| "reward": 0.3125, | |
| "reward_std": 0.2651650384068489, | |
| "rewards/reward_func": 0.3125, | |
| "step": 187 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 497.5, | |
| "epoch": 0.36953316953316956, | |
| "grad_norm": 0.6026343875240253, | |
| "kl": 0.0312042236328125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0226, | |
| "num_tokens": 3863979.0, | |
| "reward": 0.15625, | |
| "reward_std": 0.13258251920342445, | |
| "rewards/reward_func": 0.15625, | |
| "step": 188 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 547.0625, | |
| "epoch": 0.3714987714987715, | |
| "grad_norm": 0.6577878892956752, | |
| "kl": 0.030914306640625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.002, | |
| "num_tokens": 3885407.0, | |
| "reward": 0.21875, | |
| "reward_std": 0.22097086533904076, | |
| "rewards/reward_func": 0.21875, | |
| "step": 189 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 605.0625, | |
| "epoch": 0.37346437346437344, | |
| "grad_norm": 0.45127850046328344, | |
| "kl": 0.024169921875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.003, | |
| "num_tokens": 3908713.0, | |
| "reward": 0.40625, | |
| "reward_std": 0.13258251920342445, | |
| "rewards/reward_func": 0.40625, | |
| "step": 190 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 529.46875, | |
| "epoch": 0.3754299754299754, | |
| "grad_norm": 0.5469774623716551, | |
| "kl": 0.025115966796875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0056, | |
| "num_tokens": 3928856.0, | |
| "reward": 0.28125, | |
| "reward_std": 0.13258251920342445, | |
| "rewards/reward_func": 0.28125, | |
| "step": 191 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 475.15625, | |
| "epoch": 0.3773955773955774, | |
| "grad_norm": 0.9374126682104605, | |
| "kl": 0.0326385498046875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0543, | |
| "num_tokens": 3947241.0, | |
| "reward": 0.3125, | |
| "reward_std": 0.2651650384068489, | |
| "rewards/reward_func": 0.3125, | |
| "step": 192 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 547.1875, | |
| "epoch": 0.37936117936117936, | |
| "grad_norm": 0.5527175997380437, | |
| "kl": 0.0286407470703125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0571, | |
| "num_tokens": 3968765.0, | |
| "reward": 0.3125, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/reward_func": 0.3125, | |
| "step": 193 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 581.53125, | |
| "epoch": 0.38132678132678133, | |
| "grad_norm": 0.5091715028558698, | |
| "kl": 0.02716064453125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0032, | |
| "num_tokens": 3990744.0, | |
| "reward": 0.4375, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/reward_func": 0.4375, | |
| "step": 194 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 545.3125, | |
| "epoch": 0.3832923832923833, | |
| "grad_norm": 0.6906699369220881, | |
| "kl": 0.026611328125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0317, | |
| "num_tokens": 4011242.0, | |
| "reward": 0.28125, | |
| "reward_std": 0.22097086533904076, | |
| "rewards/reward_func": 0.28125, | |
| "step": 195 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 553.0625, | |
| "epoch": 0.38525798525798527, | |
| "grad_norm": 0.803923186478945, | |
| "kl": 0.026153564453125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0114, | |
| "num_tokens": 4032454.0, | |
| "reward": 0.34375, | |
| "reward_std": 0.30935921147465706, | |
| "rewards/reward_func": 0.34375, | |
| "step": 196 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 514.875, | |
| "epoch": 0.38722358722358724, | |
| "grad_norm": 0.5425045400963313, | |
| "kl": 0.029937744140625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0032, | |
| "num_tokens": 4053082.0, | |
| "reward": 0.3125, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/reward_func": 0.3125, | |
| "step": 197 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 677.78125, | |
| "epoch": 0.3891891891891892, | |
| "grad_norm": 0.6672952970651821, | |
| "kl": 0.0213623046875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0023, | |
| "num_tokens": 4078691.0, | |
| "reward": 0.125, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/reward_func": 0.125, | |
| "step": 198 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 512.03125, | |
| "epoch": 0.3911547911547912, | |
| "grad_norm": 0.476359304779961, | |
| "kl": 0.035552978515625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0066, | |
| "num_tokens": 4099720.0, | |
| "reward": 0.21875, | |
| "reward_std": 0.04419417306780815, | |
| "rewards/reward_func": 0.21875, | |
| "step": 199 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 743.375, | |
| "epoch": 0.3931203931203931, | |
| "grad_norm": 0.4679126082574762, | |
| "kl": 0.0198974609375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0029, | |
| "num_tokens": 4127002.0, | |
| "reward": 0.3125, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/reward_func": 0.3125, | |
| "step": 200 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 508, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 200, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": false | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 4, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |