| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 1.0, | |
| "eval_steps": 500, | |
| "global_step": 712, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.0014044943820224719, | |
| "grad_norm": 0.9453608393669128, | |
| "learning_rate": 6.9444444444444435e-09, | |
| "logits/chosen": -3.205078125, | |
| "logits/rejected": -3.185546875, | |
| "logps/chosen": -43.59375, | |
| "logps/rejected": -42.640625, | |
| "loss": 0.6914, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "epoch": 0.014044943820224719, | |
| "grad_norm": 1.151463508605957, | |
| "learning_rate": 6.944444444444444e-08, | |
| "logits/chosen": -3.24609375, | |
| "logits/rejected": -3.196831703186035, | |
| "logps/chosen": -42.70138931274414, | |
| "logps/rejected": -41.57638931274414, | |
| "loss": 0.6923, | |
| "rewards/accuracies": 0.2465277761220932, | |
| "rewards/chosen": 9.75396906142123e-05, | |
| "rewards/margins": -0.00013128916907589883, | |
| "rewards/rejected": 0.00022856394934933633, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.028089887640449437, | |
| "grad_norm": 0.5042410492897034, | |
| "learning_rate": 1.3888888888888888e-07, | |
| "logits/chosen": -3.255859375, | |
| "logits/rejected": -3.21484375, | |
| "logps/chosen": -42.06718826293945, | |
| "logps/rejected": -40.62968826293945, | |
| "loss": 0.6911, | |
| "rewards/accuracies": 0.30937498807907104, | |
| "rewards/chosen": 0.006805038545280695, | |
| "rewards/margins": 0.0023165703751146793, | |
| "rewards/rejected": 0.004488563630729914, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.042134831460674156, | |
| "grad_norm": 0.6343653202056885, | |
| "learning_rate": 2.0833333333333333e-07, | |
| "logits/chosen": -3.262500047683716, | |
| "logits/rejected": -3.232617139816284, | |
| "logps/chosen": -41.40625, | |
| "logps/rejected": -40.610939025878906, | |
| "loss": 0.6869, | |
| "rewards/accuracies": 0.4468750059604645, | |
| "rewards/chosen": 0.027604103088378906, | |
| "rewards/margins": 0.012437248602509499, | |
| "rewards/rejected": 0.015170956030488014, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.056179775280898875, | |
| "grad_norm": 0.7174843549728394, | |
| "learning_rate": 2.7777777777777776e-07, | |
| "logits/chosen": -3.279296875, | |
| "logits/rejected": -3.2447266578674316, | |
| "logps/chosen": -40.29218673706055, | |
| "logps/rejected": -39.842185974121094, | |
| "loss": 0.68, | |
| "rewards/accuracies": 0.5078125, | |
| "rewards/chosen": 0.03814506530761719, | |
| "rewards/margins": 0.027390670031309128, | |
| "rewards/rejected": 0.010743332095444202, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.0702247191011236, | |
| "grad_norm": 0.5838291645050049, | |
| "learning_rate": 3.472222222222222e-07, | |
| "logits/chosen": -3.2763671875, | |
| "logits/rejected": -3.230664014816284, | |
| "logps/chosen": -40.84687423706055, | |
| "logps/rejected": -40.31562423706055, | |
| "loss": 0.6619, | |
| "rewards/accuracies": 0.609375, | |
| "rewards/chosen": 0.041875459253787994, | |
| "rewards/margins": 0.06587791442871094, | |
| "rewards/rejected": -0.023966407403349876, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.08426966292134831, | |
| "grad_norm": 0.8917225003242493, | |
| "learning_rate": 4.1666666666666667e-07, | |
| "logits/chosen": -3.2505860328674316, | |
| "logits/rejected": -3.1898436546325684, | |
| "logps/chosen": -41.443748474121094, | |
| "logps/rejected": -43.939064025878906, | |
| "loss": 0.6155, | |
| "rewards/accuracies": 0.6890624761581421, | |
| "rewards/chosen": 0.011277198791503906, | |
| "rewards/margins": 0.17209243774414062, | |
| "rewards/rejected": -0.1607826203107834, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.09831460674157304, | |
| "grad_norm": 1.416306734085083, | |
| "learning_rate": 4.861111111111111e-07, | |
| "logits/chosen": -3.2109375, | |
| "logits/rejected": -3.1263670921325684, | |
| "logps/chosen": -46.9375, | |
| "logps/rejected": -53.76874923706055, | |
| "loss": 0.5336, | |
| "rewards/accuracies": 0.7015625238418579, | |
| "rewards/chosen": -0.21380920708179474, | |
| "rewards/margins": 0.407858282327652, | |
| "rewards/rejected": -0.6218963861465454, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.11235955056179775, | |
| "grad_norm": 0.6724388599395752, | |
| "learning_rate": 4.998072590601808e-07, | |
| "logits/chosen": -3.2593750953674316, | |
| "logits/rejected": -3.081835985183716, | |
| "logps/chosen": -44.157814025878906, | |
| "logps/rejected": -63.279685974121094, | |
| "loss": 0.405, | |
| "rewards/accuracies": 0.7203124761581421, | |
| "rewards/chosen": -0.117925263941288, | |
| "rewards/margins": 1.0123169422149658, | |
| "rewards/rejected": -1.1299316883087158, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.12640449438202248, | |
| "grad_norm": 0.8954005837440491, | |
| "learning_rate": 4.990247583129217e-07, | |
| "logits/chosen": -3.230273485183716, | |
| "logits/rejected": -2.9839844703674316, | |
| "logps/chosen": -45.54218673706055, | |
| "logps/rejected": -78.46562194824219, | |
| "loss": 0.3109, | |
| "rewards/accuracies": 0.7515624761581421, | |
| "rewards/chosen": -0.15528163313865662, | |
| "rewards/margins": 1.7075684070587158, | |
| "rewards/rejected": -1.863037109375, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.1404494382022472, | |
| "grad_norm": 0.5155877470970154, | |
| "learning_rate": 4.976423351108942e-07, | |
| "logits/chosen": -3.2529296875, | |
| "logits/rejected": NaN, | |
| "logps/chosen": -48.826560974121094, | |
| "logps/rejected": -96.4312515258789, | |
| "loss": 0.2579, | |
| "rewards/accuracies": 0.7562500238418579, | |
| "rewards/chosen": -0.3163391053676605, | |
| "rewards/margins": 2.467578172683716, | |
| "rewards/rejected": -2.783496141433716, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.1544943820224719, | |
| "grad_norm": 0.9068161249160767, | |
| "learning_rate": 4.95663319832678e-07, | |
| "logits/chosen": -3.236328125, | |
| "logits/rejected": -2.843554735183716, | |
| "logps/chosen": -45.657814025878906, | |
| "logps/rejected": -107.41874694824219, | |
| "loss": 0.2278, | |
| "rewards/accuracies": 0.753125011920929, | |
| "rewards/chosen": -0.1908990889787674, | |
| "rewards/margins": 3.155956983566284, | |
| "rewards/rejected": -3.34814453125, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.16853932584269662, | |
| "grad_norm": 0.5081749558448792, | |
| "learning_rate": 4.930924800994191e-07, | |
| "logits/chosen": -3.252734422683716, | |
| "logits/rejected": -2.847460985183716, | |
| "logps/chosen": -46.87812423706055, | |
| "logps/rejected": -118.3499984741211, | |
| "loss": 0.2113, | |
| "rewards/accuracies": 0.776562511920929, | |
| "rewards/chosen": -0.22010573744773865, | |
| "rewards/margins": 3.623046875, | |
| "rewards/rejected": -3.8433594703674316, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.18258426966292135, | |
| "grad_norm": 0.6637352108955383, | |
| "learning_rate": 4.899360092892144e-07, | |
| "logits/chosen": -3.2525391578674316, | |
| "logits/rejected": -2.763671875, | |
| "logps/chosen": -47.01093673706055, | |
| "logps/rejected": -124.7437515258789, | |
| "loss": 0.2106, | |
| "rewards/accuracies": 0.7671874761581421, | |
| "rewards/chosen": -0.23588410019874573, | |
| "rewards/margins": 3.9537110328674316, | |
| "rewards/rejected": -4.188086032867432, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.19662921348314608, | |
| "grad_norm": 0.3586008548736572, | |
| "learning_rate": 4.862015116167195e-07, | |
| "logits/chosen": -3.271484375, | |
| "logits/rejected": -2.750781297683716, | |
| "logps/chosen": -42.41093826293945, | |
| "logps/rejected": -132.1125030517578, | |
| "loss": 0.182, | |
| "rewards/accuracies": 0.785937488079071, | |
| "rewards/chosen": 0.008197021670639515, | |
| "rewards/margins": 4.537890434265137, | |
| "rewards/rejected": -4.529687404632568, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.21067415730337077, | |
| "grad_norm": 0.7338384985923767, | |
| "learning_rate": 4.81897983813931e-07, | |
| "logits/chosen": -3.2681641578674316, | |
| "logits/rejected": NaN, | |
| "logps/chosen": -39.69062423706055, | |
| "logps/rejected": -129.3562469482422, | |
| "loss": 0.1964, | |
| "rewards/accuracies": 0.7593749761581421, | |
| "rewards/chosen": 0.10731048882007599, | |
| "rewards/margins": 4.583203315734863, | |
| "rewards/rejected": -4.477246284484863, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.2247191011235955, | |
| "grad_norm": 0.7118776440620422, | |
| "learning_rate": 4.770357934562704e-07, | |
| "logits/chosen": -3.279296875, | |
| "logits/rejected": -2.6996092796325684, | |
| "logps/chosen": -44.157814025878906, | |
| "logps/rejected": -138.63125610351562, | |
| "loss": 0.1941, | |
| "rewards/accuracies": 0.7828124761581421, | |
| "rewards/chosen": -0.10118408501148224, | |
| "rewards/margins": 4.762890815734863, | |
| "rewards/rejected": -4.8642578125, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 0.23876404494382023, | |
| "grad_norm": 0.20937258005142212, | |
| "learning_rate": 4.716266539861866e-07, | |
| "logits/chosen": -3.1851563453674316, | |
| "logits/rejected": -2.635546922683716, | |
| "logps/chosen": -51.939064025878906, | |
| "logps/rejected": -140.15625, | |
| "loss": 0.1944, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -0.5132009387016296, | |
| "rewards/margins": 4.476171970367432, | |
| "rewards/rejected": -4.986914157867432, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 0.25280898876404495, | |
| "grad_norm": 1.1107814311981201, | |
| "learning_rate": 4.6568359649444796e-07, | |
| "logits/chosen": -3.2699217796325684, | |
| "logits/rejected": -2.663281202316284, | |
| "logps/chosen": -37.44843673706055, | |
| "logps/rejected": -139.5437469482422, | |
| "loss": 0.1877, | |
| "rewards/accuracies": 0.768750011920929, | |
| "rewards/chosen": 0.19300994277000427, | |
| "rewards/margins": 5.172265529632568, | |
| "rewards/rejected": -4.980273246765137, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 0.26685393258426965, | |
| "grad_norm": 0.1760079711675644, | |
| "learning_rate": 4.592209383271023e-07, | |
| "logits/chosen": -3.15234375, | |
| "logits/rejected": NaN, | |
| "logps/chosen": -50.7109375, | |
| "logps/rejected": -142.33749389648438, | |
| "loss": 0.2011, | |
| "rewards/accuracies": 0.753125011920929, | |
| "rewards/chosen": -0.4693801999092102, | |
| "rewards/margins": 4.672265529632568, | |
| "rewards/rejected": -5.140820503234863, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 0.2808988764044944, | |
| "grad_norm": 0.3540495038032532, | |
| "learning_rate": 4.5225424859373684e-07, | |
| "logits/chosen": -3.2144532203674316, | |
| "logits/rejected": -2.646484375, | |
| "logps/chosen": -44.23749923706055, | |
| "logps/rejected": -146.5, | |
| "loss": 0.1823, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -0.061879731714725494, | |
| "rewards/margins": 5.176171779632568, | |
| "rewards/rejected": -5.240038871765137, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.2949438202247191, | |
| "grad_norm": 0.4451713263988495, | |
| "learning_rate": 4.448003106601291e-07, | |
| "logits/chosen": -3.2412109375, | |
| "logits/rejected": -2.6431641578674316, | |
| "logps/chosen": -39.69843673706055, | |
| "logps/rejected": -141.1062469482422, | |
| "loss": 0.2101, | |
| "rewards/accuracies": 0.7671874761581421, | |
| "rewards/chosen": 0.11627502739429474, | |
| "rewards/margins": 5.123827934265137, | |
| "rewards/rejected": -5.008593559265137, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 0.3089887640449438, | |
| "grad_norm": 1.5930163860321045, | |
| "learning_rate": 4.3687708171564917e-07, | |
| "logits/chosen": -3.2291016578674316, | |
| "logits/rejected": -2.604296922683716, | |
| "logps/chosen": -42.740623474121094, | |
| "logps/rejected": -145.375, | |
| "loss": 0.2042, | |
| "rewards/accuracies": 0.7578125, | |
| "rewards/chosen": -0.070429228246212, | |
| "rewards/margins": 5.188672065734863, | |
| "rewards/rejected": -5.260156154632568, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 0.32303370786516855, | |
| "grad_norm": 0.27790066599845886, | |
| "learning_rate": 4.2850364951281705e-07, | |
| "logits/chosen": -3.2845702171325684, | |
| "logits/rejected": -2.6654295921325684, | |
| "logps/chosen": -39.85625076293945, | |
| "logps/rejected": -149.9250030517578, | |
| "loss": 0.1742, | |
| "rewards/accuracies": 0.778124988079071, | |
| "rewards/chosen": 0.12183837592601776, | |
| "rewards/margins": 5.572656154632568, | |
| "rewards/rejected": -5.451562404632568, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 0.33707865168539325, | |
| "grad_norm": 0.32155928015708923, | |
| "learning_rate": 4.1970018638323547e-07, | |
| "logits/chosen": -3.26953125, | |
| "logits/rejected": -2.6566405296325684, | |
| "logps/chosen": -41.8671875, | |
| "logps/rejected": -152.64999389648438, | |
| "loss": 0.174, | |
| "rewards/accuracies": 0.7875000238418579, | |
| "rewards/chosen": 0.051790811121463776, | |
| "rewards/margins": 5.608593940734863, | |
| "rewards/rejected": -5.559179782867432, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 0.351123595505618, | |
| "grad_norm": 0.2504253685474396, | |
| "learning_rate": 4.1048790064067573e-07, | |
| "logits/chosen": -3.254687547683716, | |
| "logits/rejected": -2.630078077316284, | |
| "logps/chosen": -40.02812576293945, | |
| "logps/rejected": -154.9250030517578, | |
| "loss": 0.1722, | |
| "rewards/accuracies": 0.785937488079071, | |
| "rewards/chosen": 0.126708984375, | |
| "rewards/margins": 5.812109470367432, | |
| "rewards/rejected": -5.681640625, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 0.3651685393258427, | |
| "grad_norm": 1.5456242561340332, | |
| "learning_rate": 4.0088898548839285e-07, | |
| "logits/chosen": -3.2398438453674316, | |
| "logits/rejected": -2.6322264671325684, | |
| "logps/chosen": -42.142189025878906, | |
| "logps/rejected": -149.5749969482422, | |
| "loss": 0.1996, | |
| "rewards/accuracies": 0.7562500238418579, | |
| "rewards/chosen": -0.08585663139820099, | |
| "rewards/margins": 5.431640625, | |
| "rewards/rejected": -5.516211032867432, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 0.3792134831460674, | |
| "grad_norm": 0.1563502699136734, | |
| "learning_rate": 3.9092656555375414e-07, | |
| "logits/chosen": -3.268359422683716, | |
| "logits/rejected": -2.6371092796325684, | |
| "logps/chosen": -41.34375, | |
| "logps/rejected": -160.625, | |
| "loss": 0.159, | |
| "rewards/accuracies": 0.796875, | |
| "rewards/chosen": 0.07463989406824112, | |
| "rewards/margins": 6.014843940734863, | |
| "rewards/rejected": -5.940234184265137, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 0.39325842696629215, | |
| "grad_norm": 0.414302796125412, | |
| "learning_rate": 3.806246411789872e-07, | |
| "logits/chosen": -3.282421827316284, | |
| "logits/rejected": NaN, | |
| "logps/chosen": -39.970314025878906, | |
| "logps/rejected": -151.84375, | |
| "loss": 0.1905, | |
| "rewards/accuracies": 0.7640625238418579, | |
| "rewards/chosen": 0.1959686279296875, | |
| "rewards/margins": 5.774609565734863, | |
| "rewards/rejected": -5.579492092132568, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 0.40730337078651685, | |
| "grad_norm": 0.11994462460279465, | |
| "learning_rate": 3.700080306022528e-07, | |
| "logits/chosen": -3.199023485183716, | |
| "logits/rejected": -2.582226514816284, | |
| "logps/chosen": -51.41093826293945, | |
| "logps/rejected": -164.3125, | |
| "loss": 0.1673, | |
| "rewards/accuracies": 0.7890625, | |
| "rewards/chosen": -0.46795958280563354, | |
| "rewards/margins": 5.684179782867432, | |
| "rewards/rejected": -6.149609565734863, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 0.42134831460674155, | |
| "grad_norm": 0.3142966330051422, | |
| "learning_rate": 3.5910231016833546e-07, | |
| "logits/chosen": -3.1810545921325684, | |
| "logits/rejected": -2.5703125, | |
| "logps/chosen": -50.90625, | |
| "logps/rejected": -161.7375030517578, | |
| "loss": 0.1775, | |
| "rewards/accuracies": 0.7796875238418579, | |
| "rewards/chosen": -0.4352920651435852, | |
| "rewards/margins": 5.612890720367432, | |
| "rewards/rejected": -6.050000190734863, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.4353932584269663, | |
| "grad_norm": 0.5336220264434814, | |
| "learning_rate": 3.4793375271298895e-07, | |
| "logits/chosen": -3.252148389816284, | |
| "logits/rejected": NaN, | |
| "logps/chosen": -41.959373474121094, | |
| "logps/rejected": -157.24374389648438, | |
| "loss": 0.1855, | |
| "rewards/accuracies": 0.768750011920929, | |
| "rewards/chosen": 0.02679443359375, | |
| "rewards/margins": 5.8720703125, | |
| "rewards/rejected": -5.84375, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 0.449438202247191, | |
| "grad_norm": 0.11426942050457001, | |
| "learning_rate": 3.3652926426937325e-07, | |
| "logits/chosen": -3.280078172683716, | |
| "logits/rejected": -2.6470704078674316, | |
| "logps/chosen": -40.0546875, | |
| "logps/rejected": -156.4812469482422, | |
| "loss": 0.1744, | |
| "rewards/accuracies": 0.770312488079071, | |
| "rewards/chosen": 0.06229095533490181, | |
| "rewards/margins": 5.884179592132568, | |
| "rewards/rejected": -5.8212890625, | |
| "step": 320 | |
| }, | |
| { | |
| "epoch": 0.46348314606741575, | |
| "grad_norm": 0.12852512300014496, | |
| "learning_rate": 3.249163192490642e-07, | |
| "logits/chosen": -3.245898485183716, | |
| "logits/rejected": -2.5863280296325684, | |
| "logps/chosen": -43.52031326293945, | |
| "logps/rejected": -163.125, | |
| "loss": 0.1654, | |
| "rewards/accuracies": 0.785937488079071, | |
| "rewards/chosen": -0.09057464450597763, | |
| "rewards/margins": 6.042578220367432, | |
| "rewards/rejected": -6.135937690734863, | |
| "step": 330 | |
| }, | |
| { | |
| "epoch": 0.47752808988764045, | |
| "grad_norm": 1.0368801355361938, | |
| "learning_rate": 3.1312289425378944e-07, | |
| "logits/chosen": -3.237109422683716, | |
| "logits/rejected": -2.5757813453674316, | |
| "logps/chosen": -45.65156173706055, | |
| "logps/rejected": -166.43124389648438, | |
| "loss": 0.1851, | |
| "rewards/accuracies": 0.7718750238418579, | |
| "rewards/chosen": -0.196772962808609, | |
| "rewards/margins": 6.102734565734863, | |
| "rewards/rejected": -6.302343845367432, | |
| "step": 340 | |
| }, | |
| { | |
| "epoch": 0.49157303370786515, | |
| "grad_norm": 0.2768089473247528, | |
| "learning_rate": 3.011774006773449e-07, | |
| "logits/chosen": -3.1851563453674316, | |
| "logits/rejected": -2.553515672683716, | |
| "logps/chosen": -50.12968826293945, | |
| "logps/rejected": -172.3125, | |
| "loss": 0.1616, | |
| "rewards/accuracies": 0.801562488079071, | |
| "rewards/chosen": -0.33214110136032104, | |
| "rewards/margins": 6.178515434265137, | |
| "rewards/rejected": -6.510156154632568, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 0.5056179775280899, | |
| "grad_norm": 0.5607307553291321, | |
| "learning_rate": 2.8910861626005773e-07, | |
| "logits/chosen": -3.2269530296325684, | |
| "logits/rejected": -2.5611329078674316, | |
| "logps/chosen": -45.09375, | |
| "logps/rejected": -164.16250610351562, | |
| "loss": 0.1745, | |
| "rewards/accuracies": 0.7718750238418579, | |
| "rewards/chosen": -0.21467895805835724, | |
| "rewards/margins": 6.003515720367432, | |
| "rewards/rejected": -6.216406345367432, | |
| "step": 360 | |
| }, | |
| { | |
| "epoch": 0.5196629213483146, | |
| "grad_norm": 0.45973560214042664, | |
| "learning_rate": 2.7694561576068983e-07, | |
| "logits/chosen": -3.203906297683716, | |
| "logits/rejected": -2.5244140625, | |
| "logps/chosen": -46.931251525878906, | |
| "logps/rejected": -171.2375030517578, | |
| "loss": 0.1627, | |
| "rewards/accuracies": 0.796875, | |
| "rewards/chosen": -0.20502586662769318, | |
| "rewards/margins": 6.284570217132568, | |
| "rewards/rejected": -6.489062309265137, | |
| "step": 370 | |
| }, | |
| { | |
| "epoch": 0.5337078651685393, | |
| "grad_norm": 0.33313286304473877, | |
| "learning_rate": 2.647177009127972e-07, | |
| "logits/chosen": -3.1996092796325684, | |
| "logits/rejected": -2.5083985328674316, | |
| "logps/chosen": -46.064064025878906, | |
| "logps/rejected": -171.14999389648438, | |
| "loss": 0.1645, | |
| "rewards/accuracies": 0.7984374761581421, | |
| "rewards/chosen": -0.1926528960466385, | |
| "rewards/margins": 6.312890529632568, | |
| "rewards/rejected": -6.504101753234863, | |
| "step": 380 | |
| }, | |
| { | |
| "epoch": 0.547752808988764, | |
| "grad_norm": 0.3406722843647003, | |
| "learning_rate": 2.524543298342874e-07, | |
| "logits/chosen": -3.216992139816284, | |
| "logits/rejected": -2.5326170921325684, | |
| "logps/chosen": -43.662498474121094, | |
| "logps/rejected": -168.7937469482422, | |
| "loss": 0.1589, | |
| "rewards/accuracies": 0.792187511920929, | |
| "rewards/chosen": -0.07064209133386612, | |
| "rewards/margins": 6.324023246765137, | |
| "rewards/rejected": -6.3935546875, | |
| "step": 390 | |
| }, | |
| { | |
| "epoch": 0.5617977528089888, | |
| "grad_norm": 0.2776348292827606, | |
| "learning_rate": 2.401850460602329e-07, | |
| "logits/chosen": -3.2457032203674316, | |
| "logits/rejected": -2.5503907203674316, | |
| "logps/chosen": -41.400001525878906, | |
| "logps/rejected": -166.88125610351562, | |
| "loss": 0.1747, | |
| "rewards/accuracies": 0.7828124761581421, | |
| "rewards/chosen": 0.02814788743853569, | |
| "rewards/margins": 6.317187309265137, | |
| "rewards/rejected": -6.288281440734863, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 0.5758426966292135, | |
| "grad_norm": 1.0503939390182495, | |
| "learning_rate": 2.2793940736990766e-07, | |
| "logits/chosen": -3.241992235183716, | |
| "logits/rejected": -2.546093702316284, | |
| "logps/chosen": -40.43437576293945, | |
| "logps/rejected": -163.3874969482422, | |
| "loss": 0.1881, | |
| "rewards/accuracies": 0.7718750238418579, | |
| "rewards/chosen": 0.005574035458266735, | |
| "rewards/margins": 6.197851657867432, | |
| "rewards/rejected": -6.190625190734863, | |
| "step": 410 | |
| }, | |
| { | |
| "epoch": 0.5898876404494382, | |
| "grad_norm": 1.4323946237564087, | |
| "learning_rate": 2.1574691457950803e-07, | |
| "logits/chosen": -3.208203077316284, | |
| "logits/rejected": NaN, | |
| "logps/chosen": -43.95781326293945, | |
| "logps/rejected": -171.10000610351562, | |
| "loss": 0.1664, | |
| "rewards/accuracies": 0.7890625, | |
| "rewards/chosen": -0.07436218112707138, | |
| "rewards/margins": 6.463086128234863, | |
| "rewards/rejected": -6.539453029632568, | |
| "step": 420 | |
| }, | |
| { | |
| "epoch": 0.6039325842696629, | |
| "grad_norm": 0.678728461265564, | |
| "learning_rate": 2.036369404721023e-07, | |
| "logits/chosen": -3.1888670921325684, | |
| "logits/rejected": NaN, | |
| "logps/chosen": -47.20624923706055, | |
| "logps/rejected": -169.1062469482422, | |
| "loss": 0.166, | |
| "rewards/accuracies": 0.778124988079071, | |
| "rewards/chosen": -0.290365606546402, | |
| "rewards/margins": 6.1806640625, | |
| "rewards/rejected": -6.469531059265137, | |
| "step": 430 | |
| }, | |
| { | |
| "epoch": 0.6179775280898876, | |
| "grad_norm": 0.20119936764240265, | |
| "learning_rate": 1.9163865903602372e-07, | |
| "logits/chosen": -3.2255859375, | |
| "logits/rejected": -2.520703077316284, | |
| "logps/chosen": -45.240623474121094, | |
| "logps/rejected": -167.2624969482422, | |
| "loss": 0.1816, | |
| "rewards/accuracies": 0.7671874761581421, | |
| "rewards/chosen": -0.21324768662452698, | |
| "rewards/margins": 6.177538871765137, | |
| "rewards/rejected": -6.390625, | |
| "step": 440 | |
| }, | |
| { | |
| "epoch": 0.6320224719101124, | |
| "grad_norm": 0.7942313551902771, | |
| "learning_rate": 1.7978097518217702e-07, | |
| "logits/chosen": -3.1763672828674316, | |
| "logits/rejected": -2.5126953125, | |
| "logps/chosen": -49.80937576293945, | |
| "logps/rejected": -172.6374969482422, | |
| "loss": 0.1685, | |
| "rewards/accuracies": 0.796875, | |
| "rewards/chosen": -0.3480590879917145, | |
| "rewards/margins": 6.218359470367432, | |
| "rewards/rejected": -6.568554878234863, | |
| "step": 450 | |
| }, | |
| { | |
| "epoch": 0.6460674157303371, | |
| "grad_norm": 0.9307264685630798, | |
| "learning_rate": 1.6809245510957666e-07, | |
| "logits/chosen": -3.2232422828674316, | |
| "logits/rejected": -2.4925780296325684, | |
| "logps/chosen": -44.875, | |
| "logps/rejected": -171.28750610351562, | |
| "loss": 0.1721, | |
| "rewards/accuracies": 0.7796875238418579, | |
| "rewards/chosen": -0.14088821411132812, | |
| "rewards/margins": 6.381640434265137, | |
| "rewards/rejected": -6.520312309265137, | |
| "step": 460 | |
| }, | |
| { | |
| "epoch": 0.6601123595505618, | |
| "grad_norm": 1.6537760496139526, | |
| "learning_rate": 1.5660125748687093e-07, | |
| "logits/chosen": -3.2333984375, | |
| "logits/rejected": -2.518749952316284, | |
| "logps/chosen": -43.334373474121094, | |
| "logps/rejected": -167.8125, | |
| "loss": 0.1794, | |
| "rewards/accuracies": 0.770312488079071, | |
| "rewards/chosen": -0.06363830715417862, | |
| "rewards/margins": 6.300976753234863, | |
| "rewards/rejected": -6.363671779632568, | |
| "step": 470 | |
| }, | |
| { | |
| "epoch": 0.6741573033707865, | |
| "grad_norm": 0.6378122568130493, | |
| "learning_rate": 1.4533506561564305e-07, | |
| "logits/chosen": -3.2416014671325684, | |
| "logits/rejected": -2.546093702316284, | |
| "logps/chosen": -39.32500076293945, | |
| "logps/rejected": -166.10000610351562, | |
| "loss": 0.1712, | |
| "rewards/accuracies": 0.7890625, | |
| "rewards/chosen": 0.16349944472312927, | |
| "rewards/margins": 6.383008003234863, | |
| "rewards/rejected": -6.220312595367432, | |
| "step": 480 | |
| }, | |
| { | |
| "epoch": 0.6882022471910112, | |
| "grad_norm": 0.21734459698200226, | |
| "learning_rate": 1.343210207389125e-07, | |
| "logits/chosen": -3.216015577316284, | |
| "logits/rejected": NaN, | |
| "logps/chosen": -39.485939025878906, | |
| "logps/rejected": -162.99374389648438, | |
| "loss": 0.1864, | |
| "rewards/accuracies": 0.7640625238418579, | |
| "rewards/chosen": 0.08720092475414276, | |
| "rewards/margins": 6.254492282867432, | |
| "rewards/rejected": -6.165625095367432, | |
| "step": 490 | |
| }, | |
| { | |
| "epoch": 0.702247191011236, | |
| "grad_norm": 0.3136639893054962, | |
| "learning_rate": 1.2358565665550387e-07, | |
| "logits/chosen": -3.216015577316284, | |
| "logits/rejected": NaN, | |
| "logps/chosen": -42.5078125, | |
| "logps/rejected": -166.36874389648438, | |
| "loss": 0.1841, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": 0.017351532354950905, | |
| "rewards/margins": 6.283398628234863, | |
| "rewards/rejected": -6.266992092132568, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 0.7162921348314607, | |
| "grad_norm": 0.07860163599252701, | |
| "learning_rate": 1.1315483579780094e-07, | |
| "logits/chosen": -3.210742235183716, | |
| "logits/rejected": -2.499218702316284, | |
| "logps/chosen": -41.615623474121094, | |
| "logps/rejected": -166.30624389648438, | |
| "loss": 0.1831, | |
| "rewards/accuracies": 0.7828124761581421, | |
| "rewards/chosen": -0.01008453406393528, | |
| "rewards/margins": 6.287304878234863, | |
| "rewards/rejected": -6.297070503234863, | |
| "step": 510 | |
| }, | |
| { | |
| "epoch": 0.7303370786516854, | |
| "grad_norm": 0.173310786485672, | |
| "learning_rate": 1.0305368692688174e-07, | |
| "logits/chosen": -3.195507764816284, | |
| "logits/rejected": -2.504101514816284, | |
| "logps/chosen": -45.17656326293945, | |
| "logps/rejected": -168.6062469482422, | |
| "loss": 0.1724, | |
| "rewards/accuracies": 0.7828124761581421, | |
| "rewards/chosen": -0.14428405463695526, | |
| "rewards/margins": 6.204297065734863, | |
| "rewards/rejected": -6.347070217132568, | |
| "step": 520 | |
| }, | |
| { | |
| "epoch": 0.7443820224719101, | |
| "grad_norm": 0.3090410828590393, | |
| "learning_rate": 9.330654459513266e-08, | |
| "logits/chosen": -3.169726610183716, | |
| "logits/rejected": -2.4736328125, | |
| "logps/chosen": -46.984375, | |
| "logps/rejected": -162.36874389648438, | |
| "loss": 0.2013, | |
| "rewards/accuracies": 0.7515624761581421, | |
| "rewards/chosen": -0.3089355528354645, | |
| "rewards/margins": 5.836718559265137, | |
| "rewards/rejected": -6.145312309265137, | |
| "step": 530 | |
| }, | |
| { | |
| "epoch": 0.7584269662921348, | |
| "grad_norm": 0.249381884932518, | |
| "learning_rate": 8.393689052217964e-08, | |
| "logits/chosen": -3.1357421875, | |
| "logits/rejected": -2.484179735183716, | |
| "logps/chosen": -49.49687576293945, | |
| "logps/rejected": -167.35000610351562, | |
| "loss": 0.1759, | |
| "rewards/accuracies": 0.770312488079071, | |
| "rewards/chosen": -0.393869012594223, | |
| "rewards/margins": 5.953320503234863, | |
| "rewards/rejected": -6.347460746765137, | |
| "step": 540 | |
| }, | |
| { | |
| "epoch": 0.7724719101123596, | |
| "grad_norm": 0.2429146021604538, | |
| "learning_rate": 7.49672970253691e-08, | |
| "logits/chosen": -3.1318359375, | |
| "logits/rejected": -2.4886717796325684, | |
| "logps/chosen": -52.857810974121094, | |
| "logps/rejected": -171.19375610351562, | |
| "loss": 0.1722, | |
| "rewards/accuracies": 0.7953125238418579, | |
| "rewards/chosen": -0.4801391661167145, | |
| "rewards/margins": 5.990429878234863, | |
| "rewards/rejected": -6.468359470367432, | |
| "step": 550 | |
| }, | |
| { | |
| "epoch": 0.7865168539325843, | |
| "grad_norm": 0.5875958204269409, | |
| "learning_rate": 6.641937264107867e-08, | |
| "logits/chosen": -3.171093702316284, | |
| "logits/rejected": -2.4808592796325684, | |
| "logps/chosen": -47.681251525878906, | |
| "logps/rejected": -174.71875, | |
| "loss": 0.1605, | |
| "rewards/accuracies": 0.7984374761581421, | |
| "rewards/chosen": -0.26392096281051636, | |
| "rewards/margins": 6.382421970367432, | |
| "rewards/rejected": -6.64453125, | |
| "step": 560 | |
| }, | |
| { | |
| "epoch": 0.800561797752809, | |
| "grad_norm": 0.37553030252456665, | |
| "learning_rate": 5.831371006785962e-08, | |
| "logits/chosen": -3.1826171875, | |
| "logits/rejected": -2.4847655296325684, | |
| "logps/chosen": -48.498435974121094, | |
| "logps/rejected": -169.1125030517578, | |
| "loss": 0.1773, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -0.3075141906738281, | |
| "rewards/margins": 6.108984470367432, | |
| "rewards/rejected": -6.417578220367432, | |
| "step": 570 | |
| }, | |
| { | |
| "epoch": 0.8146067415730337, | |
| "grad_norm": 0.31017985939979553, | |
| "learning_rate": 5.066983655682325e-08, | |
| "logits/chosen": -3.141796827316284, | |
| "logits/rejected": -2.505859375, | |
| "logps/chosen": -50.5703125, | |
| "logps/rejected": -166.83749389648438, | |
| "loss": 0.1899, | |
| "rewards/accuracies": 0.7671874761581421, | |
| "rewards/chosen": -0.4396209716796875, | |
| "rewards/margins": 5.888671875, | |
| "rewards/rejected": -6.325585842132568, | |
| "step": 580 | |
| }, | |
| { | |
| "epoch": 0.8286516853932584, | |
| "grad_norm": 0.8791071772575378, | |
| "learning_rate": 4.3506166868781755e-08, | |
| "logits/chosen": -3.1845703125, | |
| "logits/rejected": -2.4937500953674316, | |
| "logps/chosen": -48.103126525878906, | |
| "logps/rejected": -170.8625030517578, | |
| "loss": 0.1698, | |
| "rewards/accuracies": 0.785937488079071, | |
| "rewards/chosen": -0.27438658475875854, | |
| "rewards/margins": 6.2060546875, | |
| "rewards/rejected": -6.482421875, | |
| "step": 590 | |
| }, | |
| { | |
| "epoch": 0.8426966292134831, | |
| "grad_norm": 0.6104283928871155, | |
| "learning_rate": 3.683995891147695e-08, | |
| "logits/chosen": -3.176953077316284, | |
| "logits/rejected": NaN, | |
| "logps/chosen": -45.896873474121094, | |
| "logps/rejected": -168.60000610351562, | |
| "loss": 0.1721, | |
| "rewards/accuracies": 0.784375011920929, | |
| "rewards/chosen": -0.21481475234031677, | |
| "rewards/margins": 6.201952934265137, | |
| "rewards/rejected": -6.416015625, | |
| "step": 600 | |
| }, | |
| { | |
| "epoch": 0.8567415730337079, | |
| "grad_norm": 0.31073251366615295, | |
| "learning_rate": 3.0687272163768986e-08, | |
| "logits/chosen": -3.1650390625, | |
| "logits/rejected": -2.513671875, | |
| "logps/chosen": -46.66093826293945, | |
| "logps/rejected": -170.8625030517578, | |
| "loss": 0.1649, | |
| "rewards/accuracies": 0.7875000238418579, | |
| "rewards/chosen": -0.24439087510108948, | |
| "rewards/margins": 6.258593559265137, | |
| "rewards/rejected": -6.502148628234863, | |
| "step": 610 | |
| }, | |
| { | |
| "epoch": 0.8707865168539326, | |
| "grad_norm": 0.2421959638595581, | |
| "learning_rate": 2.5062928986944676e-08, | |
| "logits/chosen": -3.170117139816284, | |
| "logits/rejected": -2.5238280296325684, | |
| "logps/chosen": -46.318748474121094, | |
| "logps/rejected": -171.50625610351562, | |
| "loss": 0.1595, | |
| "rewards/accuracies": 0.7953125238418579, | |
| "rewards/chosen": -0.15370789170265198, | |
| "rewards/margins": 6.348242282867432, | |
| "rewards/rejected": -6.501953125, | |
| "step": 620 | |
| }, | |
| { | |
| "epoch": 0.8848314606741573, | |
| "grad_norm": 0.292059987783432, | |
| "learning_rate": 1.9980478916351296e-08, | |
| "logits/chosen": -3.185742139816284, | |
| "logits/rejected": -2.491992235183716, | |
| "logps/chosen": -46.803123474121094, | |
| "logps/rejected": -171.0437469482422, | |
| "loss": 0.1893, | |
| "rewards/accuracies": 0.778124988079071, | |
| "rewards/chosen": -0.1725509613752365, | |
| "rewards/margins": 6.320703029632568, | |
| "rewards/rejected": -6.491796970367432, | |
| "step": 630 | |
| }, | |
| { | |
| "epoch": 0.898876404494382, | |
| "grad_norm": 0.6732786297798157, | |
| "learning_rate": 1.5452166019378987e-08, | |
| "logits/chosen": -3.2027344703674316, | |
| "logits/rejected": -2.5123047828674316, | |
| "logps/chosen": -46.25, | |
| "logps/rejected": -172.4250030517578, | |
| "loss": 0.1712, | |
| "rewards/accuracies": 0.793749988079071, | |
| "rewards/chosen": -0.16961669921875, | |
| "rewards/margins": 6.3662109375, | |
| "rewards/rejected": -6.537890434265137, | |
| "step": 640 | |
| }, | |
| { | |
| "epoch": 0.9129213483146067, | |
| "grad_norm": 0.25237613916397095, | |
| "learning_rate": 1.1488899398429896e-08, | |
| "logits/chosen": -3.2007813453674316, | |
| "logits/rejected": -2.5160155296325684, | |
| "logps/chosen": -43.58906173706055, | |
| "logps/rejected": -170.09375, | |
| "loss": 0.1787, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -0.10042724758386612, | |
| "rewards/margins": 6.383008003234863, | |
| "rewards/rejected": -6.482421875, | |
| "step": 650 | |
| }, | |
| { | |
| "epoch": 0.9269662921348315, | |
| "grad_norm": 0.09695342183113098, | |
| "learning_rate": 8.100226909935059e-09, | |
| "logits/chosen": -3.2056641578674316, | |
| "logits/rejected": -2.5228514671325684, | |
| "logps/chosen": -43.27656173706055, | |
| "logps/rejected": -166.16250610351562, | |
| "loss": 0.1876, | |
| "rewards/accuracies": 0.7718750238418579, | |
| "rewards/chosen": -0.07465209811925888, | |
| "rewards/margins": 6.197656154632568, | |
| "rewards/rejected": -6.272265434265137, | |
| "step": 660 | |
| }, | |
| { | |
| "epoch": 0.9410112359550562, | |
| "grad_norm": 0.15045014023780823, | |
| "learning_rate": 5.2943121627319346e-09, | |
| "logits/chosen": -3.207812547683716, | |
| "logits/rejected": -2.5152344703674316, | |
| "logps/chosen": -43.06562423706055, | |
| "logps/rejected": -166.84375, | |
| "loss": 0.1818, | |
| "rewards/accuracies": 0.770312488079071, | |
| "rewards/chosen": -0.07712707668542862, | |
| "rewards/margins": 6.268164157867432, | |
| "rewards/rejected": -6.342382907867432, | |
| "step": 670 | |
| }, | |
| { | |
| "epoch": 0.9550561797752809, | |
| "grad_norm": 0.31066492199897766, | |
| "learning_rate": 3.077914851215585e-09, | |
| "logits/chosen": -3.2230467796325684, | |
| "logits/rejected": -2.5365233421325684, | |
| "logps/chosen": -44.46562576293945, | |
| "logps/rejected": -170.8625030517578, | |
| "loss": 0.1733, | |
| "rewards/accuracies": 0.7875000238418579, | |
| "rewards/chosen": -0.07046356052160263, | |
| "rewards/margins": 6.385156154632568, | |
| "rewards/rejected": -6.457812309265137, | |
| "step": 680 | |
| }, | |
| { | |
| "epoch": 0.9691011235955056, | |
| "grad_norm": 0.11322323232889175, | |
| "learning_rate": 1.4563744706429514e-09, | |
| "logits/chosen": -3.216015577316284, | |
| "logits/rejected": -2.50390625, | |
| "logps/chosen": -41.93281173706055, | |
| "logps/rejected": -168.16250610351562, | |
| "loss": 0.1693, | |
| "rewards/accuracies": 0.778124988079071, | |
| "rewards/chosen": -0.01323547400534153, | |
| "rewards/margins": 6.383593559265137, | |
| "rewards/rejected": -6.3974609375, | |
| "step": 690 | |
| }, | |
| { | |
| "epoch": 0.9831460674157303, | |
| "grad_norm": 0.28659555315971375, | |
| "learning_rate": 4.3359745382104405e-10, | |
| "logits/chosen": -3.21484375, | |
| "logits/rejected": NaN, | |
| "logps/chosen": -42.96406173706055, | |
| "logps/rejected": -169.0, | |
| "loss": 0.1708, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -0.045589447021484375, | |
| "rewards/margins": 6.397265434265137, | |
| "rewards/rejected": -6.444140434265137, | |
| "step": 700 | |
| }, | |
| { | |
| "epoch": 0.9971910112359551, | |
| "grad_norm": 0.5906934142112732, | |
| "learning_rate": 1.2047760167999133e-11, | |
| "logits/chosen": -3.2037110328674316, | |
| "logits/rejected": -2.529101610183716, | |
| "logps/chosen": -44.228126525878906, | |
| "logps/rejected": -170.83749389648438, | |
| "loss": 0.1657, | |
| "rewards/accuracies": 0.7984374761581421, | |
| "rewards/chosen": -0.09162139892578125, | |
| "rewards/margins": 6.3896484375, | |
| "rewards/rejected": -6.482421875, | |
| "step": 710 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "step": 712, | |
| "total_flos": 0.0, | |
| "train_loss": 0.23233672156092827, | |
| "train_runtime": 2773.067, | |
| "train_samples_per_second": 16.43, | |
| "train_steps_per_second": 0.257 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 712, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 500, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": false, | |
| "should_training_stop": false | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 2, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |