| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 3.0, | |
| "eval_steps": 500, | |
| "global_step": 936, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.003205128205128205, | |
| "grad_norm": 38.020048422719306, | |
| "learning_rate": 5.3191489361702125e-09, | |
| "logits/chosen": -1.484375, | |
| "logits/rejected": -1.4140625, | |
| "logps/chosen": -172.0, | |
| "logps/rejected": -160.0, | |
| "loss": 0.6914, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "epoch": 0.03205128205128205, | |
| "grad_norm": 34.89707580881474, | |
| "learning_rate": 5.3191489361702123e-08, | |
| "logits/chosen": -1.484375, | |
| "logits/rejected": -1.4375, | |
| "logps/chosen": -140.0, | |
| "logps/rejected": -181.0, | |
| "loss": 0.691, | |
| "rewards/accuracies": 0.25, | |
| "rewards/chosen": -0.00592041015625, | |
| "rewards/margins": -0.003997802734375, | |
| "rewards/rejected": -0.0019073486328125, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.0641025641025641, | |
| "grad_norm": 35.22039582592668, | |
| "learning_rate": 1.0638297872340425e-07, | |
| "logits/chosen": -1.4375, | |
| "logits/rejected": -1.40625, | |
| "logps/chosen": -116.0, | |
| "logps/rejected": -171.0, | |
| "loss": 0.6797, | |
| "rewards/accuracies": 0.48750001192092896, | |
| "rewards/chosen": 0.0260009765625, | |
| "rewards/margins": 0.0264892578125, | |
| "rewards/rejected": -0.0004749298095703125, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.09615384615384616, | |
| "grad_norm": 40.58215068588761, | |
| "learning_rate": 1.5957446808510638e-07, | |
| "logits/chosen": -1.46875, | |
| "logits/rejected": -1.4140625, | |
| "logps/chosen": -122.5, | |
| "logps/rejected": -180.0, | |
| "loss": 0.6562, | |
| "rewards/accuracies": 0.7250000238418579, | |
| "rewards/chosen": 0.11474609375, | |
| "rewards/margins": 0.08203125, | |
| "rewards/rejected": 0.03271484375, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.1282051282051282, | |
| "grad_norm": 35.55857804315714, | |
| "learning_rate": 2.127659574468085e-07, | |
| "logits/chosen": -1.53125, | |
| "logits/rejected": -1.4609375, | |
| "logps/chosen": -127.5, | |
| "logps/rejected": -180.0, | |
| "loss": 0.6092, | |
| "rewards/accuracies": 0.8125, | |
| "rewards/chosen": 0.259765625, | |
| "rewards/margins": 0.21484375, | |
| "rewards/rejected": 0.046142578125, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.16025641025641027, | |
| "grad_norm": 23.579694487139133, | |
| "learning_rate": 2.659574468085106e-07, | |
| "logits/chosen": -1.515625, | |
| "logits/rejected": -1.453125, | |
| "logps/chosen": -116.5, | |
| "logps/rejected": -174.0, | |
| "loss": 0.5579, | |
| "rewards/accuracies": 0.762499988079071, | |
| "rewards/chosen": 0.26953125, | |
| "rewards/margins": 0.365234375, | |
| "rewards/rejected": -0.095703125, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.19230769230769232, | |
| "grad_norm": 23.19132387624345, | |
| "learning_rate": 3.1914893617021275e-07, | |
| "logits/chosen": -1.453125, | |
| "logits/rejected": -1.40625, | |
| "logps/chosen": -99.5, | |
| "logps/rejected": -155.0, | |
| "loss": 0.5053, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": 0.298828125, | |
| "rewards/margins": 0.6015625, | |
| "rewards/rejected": -0.30078125, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.22435897435897437, | |
| "grad_norm": 21.818865607569602, | |
| "learning_rate": 3.7234042553191484e-07, | |
| "logits/chosen": -1.5625, | |
| "logits/rejected": -1.484375, | |
| "logps/chosen": -100.5, | |
| "logps/rejected": -162.0, | |
| "loss": 0.3894, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": 0.3984375, | |
| "rewards/margins": 1.0703125, | |
| "rewards/rejected": -0.66796875, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.2564102564102564, | |
| "grad_norm": 26.315987904129013, | |
| "learning_rate": 4.25531914893617e-07, | |
| "logits/chosen": -1.5625, | |
| "logits/rejected": -1.4921875, | |
| "logps/chosen": -117.5, | |
| "logps/rejected": -170.0, | |
| "loss": 0.3909, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": 0.026611328125, | |
| "rewards/margins": 1.1875, | |
| "rewards/rejected": -1.1640625, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.28846153846153844, | |
| "grad_norm": 28.51531224072289, | |
| "learning_rate": 4.787234042553192e-07, | |
| "logits/chosen": -1.5625, | |
| "logits/rejected": -1.515625, | |
| "logps/chosen": -114.5, | |
| "logps/rejected": -176.0, | |
| "loss": 0.387, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.1240234375, | |
| "rewards/margins": 1.375, | |
| "rewards/rejected": -1.5, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.32051282051282054, | |
| "grad_norm": 17.39940529070616, | |
| "learning_rate": 4.96437054631829e-07, | |
| "logits/chosen": -1.59375, | |
| "logits/rejected": -1.5234375, | |
| "logps/chosen": -105.5, | |
| "logps/rejected": -169.0, | |
| "loss": 0.324, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": 0.390625, | |
| "rewards/margins": 1.6640625, | |
| "rewards/rejected": -1.2734375, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.3525641025641026, | |
| "grad_norm": 28.443023292937838, | |
| "learning_rate": 4.904988123515439e-07, | |
| "logits/chosen": -1.6015625, | |
| "logits/rejected": -1.5546875, | |
| "logps/chosen": -118.0, | |
| "logps/rejected": -199.0, | |
| "loss": 0.3077, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.052490234375, | |
| "rewards/margins": 1.75, | |
| "rewards/rejected": -1.703125, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.38461538461538464, | |
| "grad_norm": 19.86248937078093, | |
| "learning_rate": 4.845605700712589e-07, | |
| "logits/chosen": -1.640625, | |
| "logits/rejected": -1.515625, | |
| "logps/chosen": -118.0, | |
| "logps/rejected": -182.0, | |
| "loss": 0.3212, | |
| "rewards/accuracies": 0.8374999761581421, | |
| "rewards/chosen": -0.41015625, | |
| "rewards/margins": 2.0625, | |
| "rewards/rejected": -2.46875, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.4166666666666667, | |
| "grad_norm": 15.34168051684897, | |
| "learning_rate": 4.786223277909738e-07, | |
| "logits/chosen": -1.625, | |
| "logits/rejected": -1.515625, | |
| "logps/chosen": -127.0, | |
| "logps/rejected": -196.0, | |
| "loss": 0.3436, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.73046875, | |
| "rewards/margins": 2.25, | |
| "rewards/rejected": -2.984375, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.44871794871794873, | |
| "grad_norm": 31.34040320177804, | |
| "learning_rate": 4.7268408551068883e-07, | |
| "logits/chosen": -1.609375, | |
| "logits/rejected": -1.515625, | |
| "logps/chosen": -138.0, | |
| "logps/rejected": -193.0, | |
| "loss": 0.335, | |
| "rewards/accuracies": 0.862500011920929, | |
| "rewards/chosen": -0.453125, | |
| "rewards/margins": 1.7578125, | |
| "rewards/rejected": -2.21875, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.4807692307692308, | |
| "grad_norm": 39.09253912133659, | |
| "learning_rate": 4.667458432304038e-07, | |
| "logits/chosen": -1.640625, | |
| "logits/rejected": -1.6328125, | |
| "logps/chosen": -128.0, | |
| "logps/rejected": -249.0, | |
| "loss": 0.2853, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.08056640625, | |
| "rewards/margins": 2.53125, | |
| "rewards/rejected": -2.609375, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.5128205128205128, | |
| "grad_norm": 17.194231471019577, | |
| "learning_rate": 4.6080760095011875e-07, | |
| "logits/chosen": -1.6015625, | |
| "logits/rejected": -1.5859375, | |
| "logps/chosen": -107.0, | |
| "logps/rejected": -191.0, | |
| "loss": 0.2666, | |
| "rewards/accuracies": 0.925000011920929, | |
| "rewards/chosen": -0.310546875, | |
| "rewards/margins": 2.71875, | |
| "rewards/rejected": -3.015625, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 0.5448717948717948, | |
| "grad_norm": 18.722209877304238, | |
| "learning_rate": 4.548693586698337e-07, | |
| "logits/chosen": -1.578125, | |
| "logits/rejected": -1.59375, | |
| "logps/chosen": -115.0, | |
| "logps/rejected": -175.0, | |
| "loss": 0.2696, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -0.150390625, | |
| "rewards/margins": 2.234375, | |
| "rewards/rejected": -2.390625, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 0.5769230769230769, | |
| "grad_norm": 21.42273705026191, | |
| "learning_rate": 4.4893111638954866e-07, | |
| "logits/chosen": -1.5859375, | |
| "logits/rejected": -1.6171875, | |
| "logps/chosen": -121.5, | |
| "logps/rejected": -203.0, | |
| "loss": 0.2899, | |
| "rewards/accuracies": 0.9125000238418579, | |
| "rewards/chosen": -0.248046875, | |
| "rewards/margins": 2.390625, | |
| "rewards/rejected": -2.640625, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 0.6089743589743589, | |
| "grad_norm": 35.105792045016955, | |
| "learning_rate": 4.429928741092636e-07, | |
| "logits/chosen": -1.625, | |
| "logits/rejected": -1.546875, | |
| "logps/chosen": -126.5, | |
| "logps/rejected": -199.0, | |
| "loss": 0.2834, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.38671875, | |
| "rewards/margins": 2.609375, | |
| "rewards/rejected": -3.0, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 0.6410256410256411, | |
| "grad_norm": 25.037051509827627, | |
| "learning_rate": 4.3705463182897863e-07, | |
| "logits/chosen": -1.59375, | |
| "logits/rejected": -1.5078125, | |
| "logps/chosen": -116.5, | |
| "logps/rejected": -171.0, | |
| "loss": 0.3022, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.59375, | |
| "rewards/margins": 2.15625, | |
| "rewards/rejected": -2.75, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.6730769230769231, | |
| "grad_norm": 25.3426239738337, | |
| "learning_rate": 4.311163895486936e-07, | |
| "logits/chosen": -1.5859375, | |
| "logits/rejected": -1.5546875, | |
| "logps/chosen": -118.5, | |
| "logps/rejected": -185.0, | |
| "loss": 0.3209, | |
| "rewards/accuracies": 0.925000011920929, | |
| "rewards/chosen": -1.21875, | |
| "rewards/margins": 2.1875, | |
| "rewards/rejected": -3.40625, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 0.7051282051282052, | |
| "grad_norm": 41.23383631897795, | |
| "learning_rate": 4.251781472684085e-07, | |
| "logits/chosen": -1.6484375, | |
| "logits/rejected": -1.5546875, | |
| "logps/chosen": -148.0, | |
| "logps/rejected": -200.0, | |
| "loss": 0.2921, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -0.609375, | |
| "rewards/margins": 2.3125, | |
| "rewards/rejected": -2.921875, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 0.7371794871794872, | |
| "grad_norm": 22.289875990424193, | |
| "learning_rate": 4.192399049881235e-07, | |
| "logits/chosen": -1.609375, | |
| "logits/rejected": -1.5859375, | |
| "logps/chosen": -127.0, | |
| "logps/rejected": -199.0, | |
| "loss": 0.2805, | |
| "rewards/accuracies": 0.925000011920929, | |
| "rewards/chosen": -0.318359375, | |
| "rewards/margins": 2.578125, | |
| "rewards/rejected": -2.90625, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 0.7692307692307693, | |
| "grad_norm": 29.25683917677673, | |
| "learning_rate": 4.1330166270783846e-07, | |
| "logits/chosen": -1.59375, | |
| "logits/rejected": -1.5390625, | |
| "logps/chosen": -108.5, | |
| "logps/rejected": -180.0, | |
| "loss": 0.2541, | |
| "rewards/accuracies": 0.9125000238418579, | |
| "rewards/chosen": -0.267578125, | |
| "rewards/margins": 2.765625, | |
| "rewards/rejected": -3.03125, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 0.8012820512820513, | |
| "grad_norm": 28.339641539576366, | |
| "learning_rate": 4.0736342042755347e-07, | |
| "logits/chosen": -1.6484375, | |
| "logits/rejected": -1.5625, | |
| "logps/chosen": -120.0, | |
| "logps/rejected": -215.0, | |
| "loss": 0.257, | |
| "rewards/accuracies": 0.887499988079071, | |
| "rewards/chosen": -0.50390625, | |
| "rewards/margins": 3.140625, | |
| "rewards/rejected": -3.640625, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 0.8333333333333334, | |
| "grad_norm": 28.458003688730955, | |
| "learning_rate": 4.0142517814726837e-07, | |
| "logits/chosen": -1.6328125, | |
| "logits/rejected": -1.5703125, | |
| "logps/chosen": -124.0, | |
| "logps/rejected": -195.0, | |
| "loss": 0.2514, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.3203125, | |
| "rewards/margins": 2.6875, | |
| "rewards/rejected": -3.015625, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 0.8653846153846154, | |
| "grad_norm": 19.712649681094366, | |
| "learning_rate": 3.9548693586698333e-07, | |
| "logits/chosen": -1.640625, | |
| "logits/rejected": -1.546875, | |
| "logps/chosen": -121.0, | |
| "logps/rejected": -177.0, | |
| "loss": 0.2635, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -0.11083984375, | |
| "rewards/margins": 2.453125, | |
| "rewards/rejected": -2.5625, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 0.8974358974358975, | |
| "grad_norm": 46.09415755147797, | |
| "learning_rate": 3.8954869358669834e-07, | |
| "logits/chosen": -1.6640625, | |
| "logits/rejected": -1.546875, | |
| "logps/chosen": -118.5, | |
| "logps/rejected": -197.0, | |
| "loss": 0.2671, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.0693359375, | |
| "rewards/margins": 2.78125, | |
| "rewards/rejected": -2.703125, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 0.9294871794871795, | |
| "grad_norm": 25.186336345995475, | |
| "learning_rate": 3.836104513064133e-07, | |
| "logits/chosen": -1.5703125, | |
| "logits/rejected": -1.5390625, | |
| "logps/chosen": -129.0, | |
| "logps/rejected": -203.0, | |
| "loss": 0.2957, | |
| "rewards/accuracies": 0.824999988079071, | |
| "rewards/chosen": -1.03125, | |
| "rewards/margins": 2.21875, | |
| "rewards/rejected": -3.25, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 0.9615384615384616, | |
| "grad_norm": 25.061643380626624, | |
| "learning_rate": 3.7767220902612825e-07, | |
| "logits/chosen": -1.609375, | |
| "logits/rejected": -1.578125, | |
| "logps/chosen": -134.0, | |
| "logps/rejected": -219.0, | |
| "loss": 0.2345, | |
| "rewards/accuracies": 0.887499988079071, | |
| "rewards/chosen": -0.90234375, | |
| "rewards/margins": 2.671875, | |
| "rewards/rejected": -3.578125, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.9935897435897436, | |
| "grad_norm": 44.17174735441787, | |
| "learning_rate": 3.717339667458432e-07, | |
| "logits/chosen": -1.640625, | |
| "logits/rejected": -1.5703125, | |
| "logps/chosen": -119.0, | |
| "logps/rejected": -192.0, | |
| "loss": 0.2734, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.2392578125, | |
| "rewards/margins": 2.734375, | |
| "rewards/rejected": -2.96875, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "eval_logits/chosen": -1.5546875, | |
| "eval_logits/rejected": -1.5234375, | |
| "eval_logps/chosen": -170.0, | |
| "eval_logps/rejected": -184.0, | |
| "eval_loss": 0.6114453077316284, | |
| "eval_rewards/accuracies": 0.5892857313156128, | |
| "eval_rewards/chosen": -1.53125, | |
| "eval_rewards/margins": 0.8984375, | |
| "eval_rewards/rejected": -2.4375, | |
| "eval_runtime": 14.9255, | |
| "eval_samples_per_second": 13.4, | |
| "eval_steps_per_second": 0.469, | |
| "step": 312 | |
| }, | |
| { | |
| "epoch": 1.0256410256410255, | |
| "grad_norm": 16.449029965031666, | |
| "learning_rate": 3.6579572446555817e-07, | |
| "logits/chosen": -1.59375, | |
| "logits/rejected": -1.578125, | |
| "logps/chosen": -117.0, | |
| "logps/rejected": -195.0, | |
| "loss": 0.1531, | |
| "rewards/accuracies": 0.925000011920929, | |
| "rewards/chosen": -0.34765625, | |
| "rewards/margins": 3.1875, | |
| "rewards/rejected": -3.546875, | |
| "step": 320 | |
| }, | |
| { | |
| "epoch": 1.0576923076923077, | |
| "grad_norm": 6.562166239087373, | |
| "learning_rate": 3.598574821852731e-07, | |
| "logits/chosen": -1.59375, | |
| "logits/rejected": -1.5703125, | |
| "logps/chosen": -111.0, | |
| "logps/rejected": -187.0, | |
| "loss": 0.1452, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.1923828125, | |
| "rewards/margins": 3.40625, | |
| "rewards/rejected": -3.609375, | |
| "step": 330 | |
| }, | |
| { | |
| "epoch": 1.0897435897435896, | |
| "grad_norm": 25.028970432454248, | |
| "learning_rate": 3.5391923990498813e-07, | |
| "logits/chosen": -1.6484375, | |
| "logits/rejected": -1.609375, | |
| "logps/chosen": -134.0, | |
| "logps/rejected": -219.0, | |
| "loss": 0.1345, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": -0.69921875, | |
| "rewards/margins": 3.609375, | |
| "rewards/rejected": -4.3125, | |
| "step": 340 | |
| }, | |
| { | |
| "epoch": 1.1217948717948718, | |
| "grad_norm": 15.596444813277682, | |
| "learning_rate": 3.479809976247031e-07, | |
| "logits/chosen": -1.65625, | |
| "logits/rejected": -1.6015625, | |
| "logps/chosen": -130.0, | |
| "logps/rejected": -207.0, | |
| "loss": 0.1092, | |
| "rewards/accuracies": 0.9624999761581421, | |
| "rewards/chosen": -0.00677490234375, | |
| "rewards/margins": 3.828125, | |
| "rewards/rejected": -3.828125, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 1.1538461538461537, | |
| "grad_norm": 22.37742199079234, | |
| "learning_rate": 3.42042755344418e-07, | |
| "logits/chosen": -1.5859375, | |
| "logits/rejected": -1.5703125, | |
| "logps/chosen": -136.0, | |
| "logps/rejected": -242.0, | |
| "loss": 0.0918, | |
| "rewards/accuracies": 0.9750000238418579, | |
| "rewards/chosen": -0.640625, | |
| "rewards/margins": 4.4375, | |
| "rewards/rejected": -5.09375, | |
| "step": 360 | |
| }, | |
| { | |
| "epoch": 1.185897435897436, | |
| "grad_norm": 27.25128799333949, | |
| "learning_rate": 3.36104513064133e-07, | |
| "logits/chosen": -1.609375, | |
| "logits/rejected": -1.59375, | |
| "logps/chosen": -154.0, | |
| "logps/rejected": -221.0, | |
| "loss": 0.1215, | |
| "rewards/accuracies": 0.9750000238418579, | |
| "rewards/chosen": -0.333984375, | |
| "rewards/margins": 4.46875, | |
| "rewards/rejected": -4.78125, | |
| "step": 370 | |
| }, | |
| { | |
| "epoch": 1.217948717948718, | |
| "grad_norm": 16.802650370434502, | |
| "learning_rate": 3.3016627078384796e-07, | |
| "logits/chosen": -1.625, | |
| "logits/rejected": -1.515625, | |
| "logps/chosen": -121.0, | |
| "logps/rejected": -184.0, | |
| "loss": 0.1051, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": -0.75390625, | |
| "rewards/margins": 3.75, | |
| "rewards/rejected": -4.5, | |
| "step": 380 | |
| }, | |
| { | |
| "epoch": 1.25, | |
| "grad_norm": 18.14599606798184, | |
| "learning_rate": 3.2422802850356297e-07, | |
| "logits/chosen": -1.65625, | |
| "logits/rejected": -1.5625, | |
| "logps/chosen": -118.0, | |
| "logps/rejected": -203.0, | |
| "loss": 0.1691, | |
| "rewards/accuracies": 0.925000011920929, | |
| "rewards/chosen": -0.4609375, | |
| "rewards/margins": 4.4375, | |
| "rewards/rejected": -4.875, | |
| "step": 390 | |
| }, | |
| { | |
| "epoch": 1.282051282051282, | |
| "grad_norm": 14.341477798166297, | |
| "learning_rate": 3.182897862232779e-07, | |
| "logits/chosen": -1.609375, | |
| "logits/rejected": -1.5546875, | |
| "logps/chosen": -112.0, | |
| "logps/rejected": -202.0, | |
| "loss": 0.1084, | |
| "rewards/accuracies": 0.9624999761581421, | |
| "rewards/chosen": -0.2578125, | |
| "rewards/margins": 4.65625, | |
| "rewards/rejected": -4.90625, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 1.314102564102564, | |
| "grad_norm": 22.538352683910524, | |
| "learning_rate": 3.1235154394299283e-07, | |
| "logits/chosen": -1.671875, | |
| "logits/rejected": -1.625, | |
| "logps/chosen": -168.0, | |
| "logps/rejected": -270.0, | |
| "loss": 0.136, | |
| "rewards/accuracies": 0.9750000238418579, | |
| "rewards/chosen": -1.1796875, | |
| "rewards/margins": 4.71875, | |
| "rewards/rejected": -5.90625, | |
| "step": 410 | |
| }, | |
| { | |
| "epoch": 1.3461538461538463, | |
| "grad_norm": 7.69063972210861, | |
| "learning_rate": 3.0641330166270784e-07, | |
| "logits/chosen": -1.609375, | |
| "logits/rejected": -1.59375, | |
| "logps/chosen": -128.0, | |
| "logps/rejected": -232.0, | |
| "loss": 0.1123, | |
| "rewards/accuracies": 0.9750000238418579, | |
| "rewards/chosen": -1.0859375, | |
| "rewards/margins": 4.59375, | |
| "rewards/rejected": -5.6875, | |
| "step": 420 | |
| }, | |
| { | |
| "epoch": 1.3782051282051282, | |
| "grad_norm": 22.81215314331503, | |
| "learning_rate": 3.004750593824228e-07, | |
| "logits/chosen": -1.6328125, | |
| "logits/rejected": -1.5390625, | |
| "logps/chosen": -122.5, | |
| "logps/rejected": -209.0, | |
| "loss": 0.0971, | |
| "rewards/accuracies": 0.9750000238418579, | |
| "rewards/chosen": -0.33984375, | |
| "rewards/margins": 4.65625, | |
| "rewards/rejected": -5.0, | |
| "step": 430 | |
| }, | |
| { | |
| "epoch": 1.4102564102564101, | |
| "grad_norm": 16.88060293661355, | |
| "learning_rate": 2.9453681710213776e-07, | |
| "logits/chosen": -1.5859375, | |
| "logits/rejected": -1.4765625, | |
| "logps/chosen": -123.0, | |
| "logps/rejected": -205.0, | |
| "loss": 0.11, | |
| "rewards/accuracies": 0.9750000238418579, | |
| "rewards/chosen": -0.4296875, | |
| "rewards/margins": 4.5625, | |
| "rewards/rejected": -5.0, | |
| "step": 440 | |
| }, | |
| { | |
| "epoch": 1.4423076923076923, | |
| "grad_norm": 14.494261083829612, | |
| "learning_rate": 2.885985748218527e-07, | |
| "logits/chosen": -1.546875, | |
| "logits/rejected": -1.5546875, | |
| "logps/chosen": -118.5, | |
| "logps/rejected": -221.0, | |
| "loss": 0.0896, | |
| "rewards/accuracies": 0.9750000238418579, | |
| "rewards/chosen": -1.0859375, | |
| "rewards/margins": 4.59375, | |
| "rewards/rejected": -5.6875, | |
| "step": 450 | |
| }, | |
| { | |
| "epoch": 1.4743589743589745, | |
| "grad_norm": 16.655718879118293, | |
| "learning_rate": 2.8266033254156767e-07, | |
| "logits/chosen": -1.59375, | |
| "logits/rejected": -1.5859375, | |
| "logps/chosen": -142.0, | |
| "logps/rejected": -256.0, | |
| "loss": 0.1071, | |
| "rewards/accuracies": 0.9750000238418579, | |
| "rewards/chosen": -1.25, | |
| "rewards/margins": 4.84375, | |
| "rewards/rejected": -6.09375, | |
| "step": 460 | |
| }, | |
| { | |
| "epoch": 1.5064102564102564, | |
| "grad_norm": 20.51874312804957, | |
| "learning_rate": 2.7672209026128263e-07, | |
| "logits/chosen": -1.5625, | |
| "logits/rejected": -1.53125, | |
| "logps/chosen": -132.0, | |
| "logps/rejected": -231.0, | |
| "loss": 0.1012, | |
| "rewards/accuracies": 0.9750000238418579, | |
| "rewards/chosen": -1.015625, | |
| "rewards/margins": 4.65625, | |
| "rewards/rejected": -5.6875, | |
| "step": 470 | |
| }, | |
| { | |
| "epoch": 1.5384615384615383, | |
| "grad_norm": 28.03019085733964, | |
| "learning_rate": 2.7078384798099764e-07, | |
| "logits/chosen": -1.4453125, | |
| "logits/rejected": -1.4609375, | |
| "logps/chosen": -104.5, | |
| "logps/rejected": -203.0, | |
| "loss": 0.1074, | |
| "rewards/accuracies": 0.987500011920929, | |
| "rewards/chosen": -1.6796875, | |
| "rewards/margins": 4.59375, | |
| "rewards/rejected": -6.3125, | |
| "step": 480 | |
| }, | |
| { | |
| "epoch": 1.5705128205128205, | |
| "grad_norm": 14.674518317046845, | |
| "learning_rate": 2.648456057007126e-07, | |
| "logits/chosen": -1.5625, | |
| "logits/rejected": -1.5390625, | |
| "logps/chosen": -139.0, | |
| "logps/rejected": -233.0, | |
| "loss": 0.1003, | |
| "rewards/accuracies": 0.987500011920929, | |
| "rewards/chosen": -1.3125, | |
| "rewards/margins": 4.9375, | |
| "rewards/rejected": -6.28125, | |
| "step": 490 | |
| }, | |
| { | |
| "epoch": 1.6025641025641026, | |
| "grad_norm": 18.731133870495857, | |
| "learning_rate": 2.589073634204275e-07, | |
| "logits/chosen": -1.5703125, | |
| "logits/rejected": -1.5625, | |
| "logps/chosen": -144.0, | |
| "logps/rejected": -251.0, | |
| "loss": 0.1228, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": -1.109375, | |
| "rewards/margins": 5.25, | |
| "rewards/rejected": -6.375, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 1.6346153846153846, | |
| "grad_norm": 16.81631331394118, | |
| "learning_rate": 2.529691211401425e-07, | |
| "logits/chosen": -1.5625, | |
| "logits/rejected": -1.5, | |
| "logps/chosen": -131.0, | |
| "logps/rejected": -230.0, | |
| "loss": 0.0819, | |
| "rewards/accuracies": 0.987500011920929, | |
| "rewards/chosen": -0.8359375, | |
| "rewards/margins": 5.375, | |
| "rewards/rejected": -6.21875, | |
| "step": 510 | |
| }, | |
| { | |
| "epoch": 1.6666666666666665, | |
| "grad_norm": 15.111001669085608, | |
| "learning_rate": 2.4703087885985747e-07, | |
| "logits/chosen": -1.546875, | |
| "logits/rejected": -1.4921875, | |
| "logps/chosen": -114.0, | |
| "logps/rejected": -202.0, | |
| "loss": 0.1091, | |
| "rewards/accuracies": 0.9750000238418579, | |
| "rewards/chosen": -0.87109375, | |
| "rewards/margins": 4.78125, | |
| "rewards/rejected": -5.625, | |
| "step": 520 | |
| }, | |
| { | |
| "epoch": 1.6987179487179487, | |
| "grad_norm": 26.75975769912583, | |
| "learning_rate": 2.410926365795724e-07, | |
| "logits/chosen": -1.5625, | |
| "logits/rejected": -1.515625, | |
| "logps/chosen": -140.0, | |
| "logps/rejected": -221.0, | |
| "loss": 0.1139, | |
| "rewards/accuracies": 0.9750000238418579, | |
| "rewards/chosen": -1.0625, | |
| "rewards/margins": 4.8125, | |
| "rewards/rejected": -5.875, | |
| "step": 530 | |
| }, | |
| { | |
| "epoch": 1.7307692307692308, | |
| "grad_norm": 13.759559493772942, | |
| "learning_rate": 2.351543942992874e-07, | |
| "logits/chosen": -1.578125, | |
| "logits/rejected": -1.5, | |
| "logps/chosen": -143.0, | |
| "logps/rejected": -229.0, | |
| "loss": 0.0978, | |
| "rewards/accuracies": 0.9750000238418579, | |
| "rewards/chosen": -0.9765625, | |
| "rewards/margins": 5.09375, | |
| "rewards/rejected": -6.0625, | |
| "step": 540 | |
| }, | |
| { | |
| "epoch": 1.7628205128205128, | |
| "grad_norm": 18.98900148314797, | |
| "learning_rate": 2.2921615201900234e-07, | |
| "logits/chosen": -1.4921875, | |
| "logits/rejected": -1.5, | |
| "logps/chosen": -114.0, | |
| "logps/rejected": -232.0, | |
| "loss": 0.1103, | |
| "rewards/accuracies": 0.9624999761581421, | |
| "rewards/chosen": -0.6953125, | |
| "rewards/margins": 5.03125, | |
| "rewards/rejected": -5.71875, | |
| "step": 550 | |
| }, | |
| { | |
| "epoch": 1.7948717948717947, | |
| "grad_norm": 7.478935072745169, | |
| "learning_rate": 2.2327790973871732e-07, | |
| "logits/chosen": -1.515625, | |
| "logits/rejected": -1.4375, | |
| "logps/chosen": -112.0, | |
| "logps/rejected": -177.0, | |
| "loss": 0.1139, | |
| "rewards/accuracies": 0.9375, | |
| "rewards/chosen": -0.6640625, | |
| "rewards/margins": 4.5625, | |
| "rewards/rejected": -5.21875, | |
| "step": 560 | |
| }, | |
| { | |
| "epoch": 1.8269230769230769, | |
| "grad_norm": 13.728806584543696, | |
| "learning_rate": 2.173396674584323e-07, | |
| "logits/chosen": -1.546875, | |
| "logits/rejected": -1.5546875, | |
| "logps/chosen": -130.0, | |
| "logps/rejected": -239.0, | |
| "loss": 0.1355, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": -0.466796875, | |
| "rewards/margins": 4.875, | |
| "rewards/rejected": -5.34375, | |
| "step": 570 | |
| }, | |
| { | |
| "epoch": 1.858974358974359, | |
| "grad_norm": 9.926742756220793, | |
| "learning_rate": 2.1140142517814726e-07, | |
| "logits/chosen": -1.609375, | |
| "logits/rejected": -1.5078125, | |
| "logps/chosen": -158.0, | |
| "logps/rejected": -202.0, | |
| "loss": 0.1118, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": -1.09375, | |
| "rewards/margins": 4.21875, | |
| "rewards/rejected": -5.3125, | |
| "step": 580 | |
| }, | |
| { | |
| "epoch": 1.891025641025641, | |
| "grad_norm": 12.431864869622972, | |
| "learning_rate": 2.0546318289786222e-07, | |
| "logits/chosen": -1.5546875, | |
| "logits/rejected": -1.53125, | |
| "logps/chosen": -122.0, | |
| "logps/rejected": -202.0, | |
| "loss": 0.1325, | |
| "rewards/accuracies": 0.987500011920929, | |
| "rewards/chosen": -0.61328125, | |
| "rewards/margins": 4.65625, | |
| "rewards/rejected": -5.28125, | |
| "step": 590 | |
| }, | |
| { | |
| "epoch": 1.9230769230769231, | |
| "grad_norm": 12.291656293105886, | |
| "learning_rate": 1.9952494061757718e-07, | |
| "logits/chosen": -1.5703125, | |
| "logits/rejected": -1.5625, | |
| "logps/chosen": -123.0, | |
| "logps/rejected": -227.0, | |
| "loss": 0.1085, | |
| "rewards/accuracies": 0.9624999761581421, | |
| "rewards/chosen": -0.67578125, | |
| "rewards/margins": 4.8125, | |
| "rewards/rejected": -5.5, | |
| "step": 600 | |
| }, | |
| { | |
| "epoch": 1.9551282051282053, | |
| "grad_norm": 6.534649119157792, | |
| "learning_rate": 1.9358669833729216e-07, | |
| "logits/chosen": -1.5546875, | |
| "logits/rejected": -1.484375, | |
| "logps/chosen": -122.0, | |
| "logps/rejected": -202.0, | |
| "loss": 0.0935, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": -0.56640625, | |
| "rewards/margins": 5.0, | |
| "rewards/rejected": -5.59375, | |
| "step": 610 | |
| }, | |
| { | |
| "epoch": 1.9871794871794872, | |
| "grad_norm": 10.91488724265466, | |
| "learning_rate": 1.876484560570071e-07, | |
| "logits/chosen": -1.5703125, | |
| "logits/rejected": -1.515625, | |
| "logps/chosen": -139.0, | |
| "logps/rejected": -236.0, | |
| "loss": 0.1071, | |
| "rewards/accuracies": 0.9624999761581421, | |
| "rewards/chosen": -1.1953125, | |
| "rewards/margins": 5.34375, | |
| "rewards/rejected": -6.53125, | |
| "step": 620 | |
| }, | |
| { | |
| "epoch": 2.0, | |
| "eval_logits/chosen": -1.5, | |
| "eval_logits/rejected": -1.46875, | |
| "eval_logps/chosen": -188.0, | |
| "eval_logps/rejected": -206.0, | |
| "eval_loss": 0.700976550579071, | |
| "eval_rewards/accuracies": 0.6428571343421936, | |
| "eval_rewards/chosen": -3.296875, | |
| "eval_rewards/margins": 1.3046875, | |
| "eval_rewards/rejected": -4.59375, | |
| "eval_runtime": 15.0135, | |
| "eval_samples_per_second": 13.321, | |
| "eval_steps_per_second": 0.466, | |
| "step": 624 | |
| }, | |
| { | |
| "epoch": 2.019230769230769, | |
| "grad_norm": 2.9708656343687063, | |
| "learning_rate": 1.8171021377672207e-07, | |
| "logits/chosen": -1.515625, | |
| "logits/rejected": -1.5, | |
| "logps/chosen": -150.0, | |
| "logps/rejected": -224.0, | |
| "loss": 0.071, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": -0.98828125, | |
| "rewards/margins": 5.40625, | |
| "rewards/rejected": -6.40625, | |
| "step": 630 | |
| }, | |
| { | |
| "epoch": 2.051282051282051, | |
| "grad_norm": 3.1451139864714266, | |
| "learning_rate": 1.7577197149643706e-07, | |
| "logits/chosen": -1.5078125, | |
| "logits/rejected": -1.4609375, | |
| "logps/chosen": -121.5, | |
| "logps/rejected": -198.0, | |
| "loss": 0.0534, | |
| "rewards/accuracies": 0.9624999761581421, | |
| "rewards/chosen": -0.8203125, | |
| "rewards/margins": 4.9375, | |
| "rewards/rejected": -5.75, | |
| "step": 640 | |
| }, | |
| { | |
| "epoch": 2.0833333333333335, | |
| "grad_norm": 7.314670214652841, | |
| "learning_rate": 1.6983372921615202e-07, | |
| "logits/chosen": -1.5703125, | |
| "logits/rejected": -1.546875, | |
| "logps/chosen": -134.0, | |
| "logps/rejected": -247.0, | |
| "loss": 0.0546, | |
| "rewards/accuracies": 0.987500011920929, | |
| "rewards/chosen": -0.73046875, | |
| "rewards/margins": 5.84375, | |
| "rewards/rejected": -6.5625, | |
| "step": 650 | |
| }, | |
| { | |
| "epoch": 2.1153846153846154, | |
| "grad_norm": 6.986967407965111, | |
| "learning_rate": 1.6389548693586697e-07, | |
| "logits/chosen": -1.5625, | |
| "logits/rejected": -1.53125, | |
| "logps/chosen": -123.5, | |
| "logps/rejected": -219.0, | |
| "loss": 0.0485, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.9375, | |
| "rewards/margins": 5.5625, | |
| "rewards/rejected": -6.53125, | |
| "step": 660 | |
| }, | |
| { | |
| "epoch": 2.1474358974358974, | |
| "grad_norm": 3.152252590941172, | |
| "learning_rate": 1.5795724465558193e-07, | |
| "logits/chosen": -1.515625, | |
| "logits/rejected": -1.53125, | |
| "logps/chosen": -121.5, | |
| "logps/rejected": -264.0, | |
| "loss": 0.0391, | |
| "rewards/accuracies": 0.987500011920929, | |
| "rewards/chosen": -0.97265625, | |
| "rewards/margins": 6.40625, | |
| "rewards/rejected": -7.375, | |
| "step": 670 | |
| }, | |
| { | |
| "epoch": 2.1794871794871793, | |
| "grad_norm": 5.616933414198294, | |
| "learning_rate": 1.520190023752969e-07, | |
| "logits/chosen": -1.5390625, | |
| "logits/rejected": -1.5078125, | |
| "logps/chosen": -136.0, | |
| "logps/rejected": -225.0, | |
| "loss": 0.0523, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": -1.265625, | |
| "rewards/margins": 5.4375, | |
| "rewards/rejected": -6.6875, | |
| "step": 680 | |
| }, | |
| { | |
| "epoch": 2.2115384615384617, | |
| "grad_norm": 4.462570602831943, | |
| "learning_rate": 1.4608076009501184e-07, | |
| "logits/chosen": -1.5390625, | |
| "logits/rejected": -1.53125, | |
| "logps/chosen": -141.0, | |
| "logps/rejected": -247.0, | |
| "loss": 0.0548, | |
| "rewards/accuracies": 0.9624999761581421, | |
| "rewards/chosen": -0.9921875, | |
| "rewards/margins": 5.90625, | |
| "rewards/rejected": -6.90625, | |
| "step": 690 | |
| }, | |
| { | |
| "epoch": 2.2435897435897436, | |
| "grad_norm": 6.786342065093859, | |
| "learning_rate": 1.4014251781472683e-07, | |
| "logits/chosen": -1.5546875, | |
| "logits/rejected": -1.546875, | |
| "logps/chosen": -115.0, | |
| "logps/rejected": -243.0, | |
| "loss": 0.0328, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.7890625, | |
| "rewards/margins": 6.25, | |
| "rewards/rejected": -7.03125, | |
| "step": 700 | |
| }, | |
| { | |
| "epoch": 2.2756410256410255, | |
| "grad_norm": 10.783463457904992, | |
| "learning_rate": 1.342042755344418e-07, | |
| "logits/chosen": -1.5390625, | |
| "logits/rejected": -1.546875, | |
| "logps/chosen": -149.0, | |
| "logps/rejected": -266.0, | |
| "loss": 0.0556, | |
| "rewards/accuracies": 0.9750000238418579, | |
| "rewards/chosen": -1.203125, | |
| "rewards/margins": 6.125, | |
| "rewards/rejected": -7.3125, | |
| "step": 710 | |
| }, | |
| { | |
| "epoch": 2.3076923076923075, | |
| "grad_norm": 9.0679334215508, | |
| "learning_rate": 1.2826603325415677e-07, | |
| "logits/chosen": -1.53125, | |
| "logits/rejected": -1.4765625, | |
| "logps/chosen": -121.0, | |
| "logps/rejected": -217.0, | |
| "loss": 0.0438, | |
| "rewards/accuracies": 0.987500011920929, | |
| "rewards/chosen": -0.80078125, | |
| "rewards/margins": 5.84375, | |
| "rewards/rejected": -6.65625, | |
| "step": 720 | |
| }, | |
| { | |
| "epoch": 2.33974358974359, | |
| "grad_norm": 2.164285473082669, | |
| "learning_rate": 1.2232779097387173e-07, | |
| "logits/chosen": -1.5078125, | |
| "logits/rejected": -1.5, | |
| "logps/chosen": -153.0, | |
| "logps/rejected": -237.0, | |
| "loss": 0.0414, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -1.2421875, | |
| "rewards/margins": 6.15625, | |
| "rewards/rejected": -7.375, | |
| "step": 730 | |
| }, | |
| { | |
| "epoch": 2.371794871794872, | |
| "grad_norm": 8.601824238964523, | |
| "learning_rate": 1.163895486935867e-07, | |
| "logits/chosen": -1.4921875, | |
| "logits/rejected": -1.46875, | |
| "logps/chosen": -148.0, | |
| "logps/rejected": -238.0, | |
| "loss": 0.0358, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -1.6953125, | |
| "rewards/margins": 5.9375, | |
| "rewards/rejected": -7.625, | |
| "step": 740 | |
| }, | |
| { | |
| "epoch": 2.4038461538461537, | |
| "grad_norm": 11.674320163023708, | |
| "learning_rate": 1.1045130641330165e-07, | |
| "logits/chosen": -1.5234375, | |
| "logits/rejected": -1.5390625, | |
| "logps/chosen": -133.0, | |
| "logps/rejected": -258.0, | |
| "loss": 0.0523, | |
| "rewards/accuracies": 0.987500011920929, | |
| "rewards/chosen": -1.9453125, | |
| "rewards/margins": 6.4375, | |
| "rewards/rejected": -8.375, | |
| "step": 750 | |
| }, | |
| { | |
| "epoch": 2.435897435897436, | |
| "grad_norm": 4.6040374115250415, | |
| "learning_rate": 1.0451306413301662e-07, | |
| "logits/chosen": -1.515625, | |
| "logits/rejected": -1.5234375, | |
| "logps/chosen": -115.5, | |
| "logps/rejected": -258.0, | |
| "loss": 0.0425, | |
| "rewards/accuracies": 0.987500011920929, | |
| "rewards/chosen": -0.984375, | |
| "rewards/margins": 7.40625, | |
| "rewards/rejected": -8.375, | |
| "step": 760 | |
| }, | |
| { | |
| "epoch": 2.467948717948718, | |
| "grad_norm": 6.059248925740205, | |
| "learning_rate": 9.857482185273158e-08, | |
| "logits/chosen": -1.53125, | |
| "logits/rejected": -1.515625, | |
| "logps/chosen": -137.0, | |
| "logps/rejected": -233.0, | |
| "loss": 0.037, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -1.2265625, | |
| "rewards/margins": 5.90625, | |
| "rewards/rejected": -7.125, | |
| "step": 770 | |
| }, | |
| { | |
| "epoch": 2.5, | |
| "grad_norm": 5.569807408041334, | |
| "learning_rate": 9.263657957244655e-08, | |
| "logits/chosen": -1.546875, | |
| "logits/rejected": -1.53125, | |
| "logps/chosen": -136.0, | |
| "logps/rejected": -240.0, | |
| "loss": 0.0383, | |
| "rewards/accuracies": 0.987500011920929, | |
| "rewards/chosen": -1.46875, | |
| "rewards/margins": 6.78125, | |
| "rewards/rejected": -8.25, | |
| "step": 780 | |
| }, | |
| { | |
| "epoch": 2.532051282051282, | |
| "grad_norm": 9.080566390829592, | |
| "learning_rate": 8.669833729216151e-08, | |
| "logits/chosen": -1.515625, | |
| "logits/rejected": -1.4375, | |
| "logps/chosen": -138.0, | |
| "logps/rejected": -233.0, | |
| "loss": 0.043, | |
| "rewards/accuracies": 0.987500011920929, | |
| "rewards/chosen": -2.125, | |
| "rewards/margins": 6.1875, | |
| "rewards/rejected": -8.3125, | |
| "step": 790 | |
| }, | |
| { | |
| "epoch": 2.564102564102564, | |
| "grad_norm": 4.722752417362213, | |
| "learning_rate": 8.076009501187649e-08, | |
| "logits/chosen": -1.4609375, | |
| "logits/rejected": -1.515625, | |
| "logps/chosen": -129.0, | |
| "logps/rejected": -244.0, | |
| "loss": 0.043, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -1.78125, | |
| "rewards/margins": 6.5625, | |
| "rewards/rejected": -8.375, | |
| "step": 800 | |
| }, | |
| { | |
| "epoch": 2.5961538461538463, | |
| "grad_norm": 8.320380078890436, | |
| "learning_rate": 7.482185273159145e-08, | |
| "logits/chosen": -1.5625, | |
| "logits/rejected": -1.484375, | |
| "logps/chosen": -152.0, | |
| "logps/rejected": -234.0, | |
| "loss": 0.0311, | |
| "rewards/accuracies": 0.987500011920929, | |
| "rewards/chosen": -1.734375, | |
| "rewards/margins": 6.25, | |
| "rewards/rejected": -7.96875, | |
| "step": 810 | |
| }, | |
| { | |
| "epoch": 2.628205128205128, | |
| "grad_norm": 10.713886798327927, | |
| "learning_rate": 6.88836104513064e-08, | |
| "logits/chosen": -1.5078125, | |
| "logits/rejected": -1.4765625, | |
| "logps/chosen": -151.0, | |
| "logps/rejected": -252.0, | |
| "loss": 0.0368, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -1.6875, | |
| "rewards/margins": 6.25, | |
| "rewards/rejected": -7.9375, | |
| "step": 820 | |
| }, | |
| { | |
| "epoch": 2.66025641025641, | |
| "grad_norm": 10.956365394123852, | |
| "learning_rate": 6.294536817102138e-08, | |
| "logits/chosen": -1.484375, | |
| "logits/rejected": -1.4375, | |
| "logps/chosen": -139.0, | |
| "logps/rejected": -224.0, | |
| "loss": 0.0434, | |
| "rewards/accuracies": 0.9750000238418579, | |
| "rewards/chosen": -1.7421875, | |
| "rewards/margins": 6.15625, | |
| "rewards/rejected": -7.875, | |
| "step": 830 | |
| }, | |
| { | |
| "epoch": 2.6923076923076925, | |
| "grad_norm": 5.167870965947855, | |
| "learning_rate": 5.700712589073634e-08, | |
| "logits/chosen": -1.5546875, | |
| "logits/rejected": -1.484375, | |
| "logps/chosen": -155.0, | |
| "logps/rejected": -286.0, | |
| "loss": 0.0367, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -1.4140625, | |
| "rewards/margins": 7.15625, | |
| "rewards/rejected": -8.5625, | |
| "step": 840 | |
| }, | |
| { | |
| "epoch": 2.7243589743589745, | |
| "grad_norm": 10.357396384526298, | |
| "learning_rate": 5.10688836104513e-08, | |
| "logits/chosen": -1.4765625, | |
| "logits/rejected": -1.5, | |
| "logps/chosen": -144.0, | |
| "logps/rejected": -245.0, | |
| "loss": 0.0381, | |
| "rewards/accuracies": 0.987500011920929, | |
| "rewards/chosen": -1.8125, | |
| "rewards/margins": 6.46875, | |
| "rewards/rejected": -8.3125, | |
| "step": 850 | |
| }, | |
| { | |
| "epoch": 2.7564102564102564, | |
| "grad_norm": 4.157699399133391, | |
| "learning_rate": 4.5130641330166267e-08, | |
| "logits/chosen": -1.46875, | |
| "logits/rejected": -1.421875, | |
| "logps/chosen": -126.0, | |
| "logps/rejected": -239.0, | |
| "loss": 0.0442, | |
| "rewards/accuracies": 0.9375, | |
| "rewards/chosen": -1.5859375, | |
| "rewards/margins": 6.25, | |
| "rewards/rejected": -7.8125, | |
| "step": 860 | |
| }, | |
| { | |
| "epoch": 2.7884615384615383, | |
| "grad_norm": 10.34195664309316, | |
| "learning_rate": 3.919239904988123e-08, | |
| "logits/chosen": -1.5234375, | |
| "logits/rejected": -1.5390625, | |
| "logps/chosen": -148.0, | |
| "logps/rejected": -268.0, | |
| "loss": 0.036, | |
| "rewards/accuracies": 0.987500011920929, | |
| "rewards/chosen": -2.296875, | |
| "rewards/margins": 6.5, | |
| "rewards/rejected": -8.8125, | |
| "step": 870 | |
| }, | |
| { | |
| "epoch": 2.8205128205128203, | |
| "grad_norm": 13.885196758276543, | |
| "learning_rate": 3.32541567695962e-08, | |
| "logits/chosen": -1.53125, | |
| "logits/rejected": -1.4609375, | |
| "logps/chosen": -139.0, | |
| "logps/rejected": -237.0, | |
| "loss": 0.0386, | |
| "rewards/accuracies": 0.987500011920929, | |
| "rewards/chosen": -1.7890625, | |
| "rewards/margins": 6.4375, | |
| "rewards/rejected": -8.25, | |
| "step": 880 | |
| }, | |
| { | |
| "epoch": 2.8525641025641026, | |
| "grad_norm": 3.298992157382736, | |
| "learning_rate": 2.7315914489311164e-08, | |
| "logits/chosen": -1.53125, | |
| "logits/rejected": -1.484375, | |
| "logps/chosen": -142.0, | |
| "logps/rejected": -264.0, | |
| "loss": 0.0326, | |
| "rewards/accuracies": 0.987500011920929, | |
| "rewards/chosen": -2.0, | |
| "rewards/margins": 6.84375, | |
| "rewards/rejected": -8.875, | |
| "step": 890 | |
| }, | |
| { | |
| "epoch": 2.8846153846153846, | |
| "grad_norm": 9.55680354164709, | |
| "learning_rate": 2.1377672209026125e-08, | |
| "logits/chosen": -1.4609375, | |
| "logits/rejected": -1.4453125, | |
| "logps/chosen": -124.0, | |
| "logps/rejected": -247.0, | |
| "loss": 0.0386, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -1.7734375, | |
| "rewards/margins": 7.0, | |
| "rewards/rejected": -8.8125, | |
| "step": 900 | |
| }, | |
| { | |
| "epoch": 2.9166666666666665, | |
| "grad_norm": 8.674386112287156, | |
| "learning_rate": 1.5439429928741092e-08, | |
| "logits/chosen": -1.5078125, | |
| "logits/rejected": -1.4765625, | |
| "logps/chosen": -118.0, | |
| "logps/rejected": -240.0, | |
| "loss": 0.0517, | |
| "rewards/accuracies": 0.987500011920929, | |
| "rewards/chosen": -1.3984375, | |
| "rewards/margins": 7.0625, | |
| "rewards/rejected": -8.4375, | |
| "step": 910 | |
| }, | |
| { | |
| "epoch": 2.948717948717949, | |
| "grad_norm": 6.427610044394737, | |
| "learning_rate": 9.501187648456057e-09, | |
| "logits/chosen": -1.5, | |
| "logits/rejected": -1.4765625, | |
| "logps/chosen": -148.0, | |
| "logps/rejected": -256.0, | |
| "loss": 0.0481, | |
| "rewards/accuracies": 0.987500011920929, | |
| "rewards/chosen": -1.84375, | |
| "rewards/margins": 6.375, | |
| "rewards/rejected": -8.25, | |
| "step": 920 | |
| }, | |
| { | |
| "epoch": 2.980769230769231, | |
| "grad_norm": 20.775958318465445, | |
| "learning_rate": 3.562945368171021e-09, | |
| "logits/chosen": -1.53125, | |
| "logits/rejected": -1.5, | |
| "logps/chosen": -141.0, | |
| "logps/rejected": -232.0, | |
| "loss": 0.0498, | |
| "rewards/accuracies": 0.9624999761581421, | |
| "rewards/chosen": -1.640625, | |
| "rewards/margins": 6.0, | |
| "rewards/rejected": -7.625, | |
| "step": 930 | |
| }, | |
| { | |
| "epoch": 3.0, | |
| "eval_logits/chosen": -1.4921875, | |
| "eval_logits/rejected": -1.4609375, | |
| "eval_logps/chosen": -199.0, | |
| "eval_logps/rejected": -222.0, | |
| "eval_loss": 0.830273449420929, | |
| "eval_rewards/accuracies": 0.625, | |
| "eval_rewards/chosen": -4.4375, | |
| "eval_rewards/margins": 1.765625, | |
| "eval_rewards/rejected": -6.1875, | |
| "eval_runtime": 16.7003, | |
| "eval_samples_per_second": 11.976, | |
| "eval_steps_per_second": 0.419, | |
| "step": 936 | |
| }, | |
| { | |
| "epoch": 3.0, | |
| "step": 936, | |
| "total_flos": 0.0, | |
| "train_loss": 0.1719489719113733, | |
| "train_runtime": 6022.2267, | |
| "train_samples_per_second": 4.969, | |
| "train_steps_per_second": 0.155 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 936, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 3, | |
| "save_steps": 500, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 8, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |