| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 1.0, | |
| "eval_steps": 500, | |
| "global_step": 542, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.0018450184501845018, | |
| "grad_norm": 216.73105815307596, | |
| "learning_rate": 9.09090909090909e-09, | |
| "logits/chosen": 0.7109375, | |
| "logits/rejected": 0.52734375, | |
| "logps/chosen": -508.0, | |
| "logps/rejected": -644.0, | |
| "loss": 0.6914, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "epoch": 0.01845018450184502, | |
| "grad_norm": 210.44033190764196, | |
| "learning_rate": 9.09090909090909e-08, | |
| "logits/chosen": 0.44140625, | |
| "logits/rejected": 0.15234375, | |
| "logps/chosen": -432.0, | |
| "logps/rejected": -324.0, | |
| "loss": 0.7003, | |
| "rewards/accuracies": 0.1111111119389534, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.022216796875, | |
| "rewards/rejected": -0.022216796875, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.03690036900369004, | |
| "grad_norm": 209.35372353617072, | |
| "learning_rate": 1.818181818181818e-07, | |
| "logits/chosen": 0.4375, | |
| "logits/rejected": 0.1962890625, | |
| "logps/chosen": -404.0, | |
| "logps/rejected": -400.0, | |
| "loss": 0.6372, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": 0.169921875, | |
| "rewards/margins": 0.265625, | |
| "rewards/rejected": -0.09521484375, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.055350553505535055, | |
| "grad_norm": 273.04922910431503, | |
| "learning_rate": 2.727272727272727e-07, | |
| "logits/chosen": 0.76171875, | |
| "logits/rejected": 0.65625, | |
| "logps/chosen": -440.0, | |
| "logps/rejected": -426.0, | |
| "loss": 0.6013, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": 0.2060546875, | |
| "rewards/margins": 0.390625, | |
| "rewards/rejected": -0.185546875, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.07380073800738007, | |
| "grad_norm": 104.08445146547913, | |
| "learning_rate": 3.636363636363636e-07, | |
| "logits/chosen": 0.353515625, | |
| "logits/rejected": 0.390625, | |
| "logps/chosen": -404.0, | |
| "logps/rejected": -386.0, | |
| "loss": 0.4319, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": 0.259765625, | |
| "rewards/margins": 0.8984375, | |
| "rewards/rejected": -0.640625, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.09225092250922509, | |
| "grad_norm": 218.12945883654578, | |
| "learning_rate": 4.545454545454545e-07, | |
| "logits/chosen": 0.58984375, | |
| "logits/rejected": 0.69140625, | |
| "logps/chosen": -680.0, | |
| "logps/rejected": -456.0, | |
| "loss": 0.2879, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.49609375, | |
| "rewards/margins": 2.46875, | |
| "rewards/rejected": -1.9765625, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.11070110701107011, | |
| "grad_norm": 73.18005599526808, | |
| "learning_rate": 4.948665297741273e-07, | |
| "logits/chosen": 0.50390625, | |
| "logits/rejected": 0.267578125, | |
| "logps/chosen": -476.0, | |
| "logps/rejected": -496.0, | |
| "loss": 0.2546, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -0.474609375, | |
| "rewards/margins": 2.21875, | |
| "rewards/rejected": -2.703125, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.12915129151291513, | |
| "grad_norm": 88.23967604073839, | |
| "learning_rate": 4.845995893223819e-07, | |
| "logits/chosen": 1.0703125, | |
| "logits/rejected": 0.90234375, | |
| "logps/chosen": -388.0, | |
| "logps/rejected": -410.0, | |
| "loss": 0.222, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": -0.9609375, | |
| "rewards/margins": 4.0, | |
| "rewards/rejected": -4.96875, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.14760147601476015, | |
| "grad_norm": 79.9788314581183, | |
| "learning_rate": 4.743326488706365e-07, | |
| "logits/chosen": 0.36328125, | |
| "logits/rejected": 0.384765625, | |
| "logps/chosen": -468.0, | |
| "logps/rejected": -488.0, | |
| "loss": 0.2265, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -1.171875, | |
| "rewards/margins": 4.59375, | |
| "rewards/rejected": -5.75, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.16605166051660517, | |
| "grad_norm": 43.19506066608376, | |
| "learning_rate": 4.6406570841889114e-07, | |
| "logits/chosen": 0.58203125, | |
| "logits/rejected": 0.5703125, | |
| "logps/chosen": -482.0, | |
| "logps/rejected": -568.0, | |
| "loss": 0.2071, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": -1.3984375, | |
| "rewards/margins": 4.03125, | |
| "rewards/rejected": -5.4375, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.18450184501845018, | |
| "grad_norm": 35.346905504143315, | |
| "learning_rate": 4.537987679671458e-07, | |
| "logits/chosen": 0.458984375, | |
| "logits/rejected": 0.55859375, | |
| "logps/chosen": -516.0, | |
| "logps/rejected": -544.0, | |
| "loss": 0.0814, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -2.046875, | |
| "rewards/margins": 7.34375, | |
| "rewards/rejected": -9.375, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.2029520295202952, | |
| "grad_norm": 245.44538831203513, | |
| "learning_rate": 4.435318275154004e-07, | |
| "logits/chosen": 0.7734375, | |
| "logits/rejected": 0.78515625, | |
| "logps/chosen": -384.0, | |
| "logps/rejected": -458.0, | |
| "loss": 0.1581, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -3.90625, | |
| "rewards/margins": 6.28125, | |
| "rewards/rejected": -10.1875, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.22140221402214022, | |
| "grad_norm": 33.61709141902719, | |
| "learning_rate": 4.33264887063655e-07, | |
| "logits/chosen": 0.37890625, | |
| "logits/rejected": 0.05322265625, | |
| "logps/chosen": -502.0, | |
| "logps/rejected": -744.0, | |
| "loss": 0.0645, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": -3.703125, | |
| "rewards/margins": 11.375, | |
| "rewards/rejected": -15.0625, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.23985239852398524, | |
| "grad_norm": 86.40618808592508, | |
| "learning_rate": 4.2299794661190965e-07, | |
| "logits/chosen": 0.494140625, | |
| "logits/rejected": 0.71875, | |
| "logps/chosen": -548.0, | |
| "logps/rejected": -564.0, | |
| "loss": 0.1139, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": -4.46875, | |
| "rewards/margins": 9.5, | |
| "rewards/rejected": -14.0, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.25830258302583026, | |
| "grad_norm": 50.835987084584914, | |
| "learning_rate": 4.1273100616016426e-07, | |
| "logits/chosen": 0.71875, | |
| "logits/rejected": 0.5234375, | |
| "logps/chosen": -428.0, | |
| "logps/rejected": -652.0, | |
| "loss": 0.0664, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": -4.75, | |
| "rewards/margins": 10.5625, | |
| "rewards/rejected": -15.3125, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.2767527675276753, | |
| "grad_norm": 5.946935308063434, | |
| "learning_rate": 4.024640657084189e-07, | |
| "logits/chosen": 0.984375, | |
| "logits/rejected": 0.83984375, | |
| "logps/chosen": -478.0, | |
| "logps/rejected": -648.0, | |
| "loss": 0.069, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": -6.46875, | |
| "rewards/margins": 9.5625, | |
| "rewards/rejected": -16.0, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.2952029520295203, | |
| "grad_norm": 0.1622108188450815, | |
| "learning_rate": 3.9219712525667346e-07, | |
| "logits/chosen": 1.203125, | |
| "logits/rejected": 0.7421875, | |
| "logps/chosen": -472.0, | |
| "logps/rejected": -744.0, | |
| "loss": 0.0452, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": -6.375, | |
| "rewards/margins": 10.875, | |
| "rewards/rejected": -17.25, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 0.31365313653136534, | |
| "grad_norm": 6.158419137937965, | |
| "learning_rate": 3.819301848049281e-07, | |
| "logits/chosen": 0.80859375, | |
| "logits/rejected": 0.56640625, | |
| "logps/chosen": -446.0, | |
| "logps/rejected": -732.0, | |
| "loss": 0.157, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -7.0, | |
| "rewards/margins": 11.4375, | |
| "rewards/rejected": -18.5, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 0.33210332103321033, | |
| "grad_norm": 0.13813545629632493, | |
| "learning_rate": 3.7166324435318277e-07, | |
| "logits/chosen": 0.9296875, | |
| "logits/rejected": 0.79296875, | |
| "logps/chosen": -498.0, | |
| "logps/rejected": -732.0, | |
| "loss": 0.093, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -6.875, | |
| "rewards/margins": 12.9375, | |
| "rewards/rejected": -19.75, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 0.3505535055350554, | |
| "grad_norm": 2.690722962554207, | |
| "learning_rate": 3.6139630390143737e-07, | |
| "logits/chosen": 0.431640625, | |
| "logits/rejected": 0.578125, | |
| "logps/chosen": -624.0, | |
| "logps/rejected": -656.0, | |
| "loss": 0.0374, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -8.125, | |
| "rewards/margins": 8.875, | |
| "rewards/rejected": -17.0, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 0.36900369003690037, | |
| "grad_norm": 7.595277330508933, | |
| "learning_rate": 3.51129363449692e-07, | |
| "logits/chosen": 0.9140625, | |
| "logits/rejected": 0.6328125, | |
| "logps/chosen": -480.0, | |
| "logps/rejected": -552.0, | |
| "loss": 0.079, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -8.625, | |
| "rewards/margins": 8.5, | |
| "rewards/rejected": -17.125, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.3874538745387454, | |
| "grad_norm": 0.3720122163023151, | |
| "learning_rate": 3.408624229979466e-07, | |
| "logits/chosen": 0.275390625, | |
| "logits/rejected": 0.10009765625, | |
| "logps/chosen": -776.0, | |
| "logps/rejected": -792.0, | |
| "loss": 0.0949, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -8.125, | |
| "rewards/margins": 11.5, | |
| "rewards/rejected": -19.625, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 0.4059040590405904, | |
| "grad_norm": 5.792747044896083, | |
| "learning_rate": 3.3059548254620123e-07, | |
| "logits/chosen": 0.62890625, | |
| "logits/rejected": 0.5625, | |
| "logps/chosen": -466.0, | |
| "logps/rejected": -620.0, | |
| "loss": 0.0396, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": -8.375, | |
| "rewards/margins": 9.8125, | |
| "rewards/rejected": -18.25, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 0.42435424354243545, | |
| "grad_norm": 4.444541488962383, | |
| "learning_rate": 3.2032854209445584e-07, | |
| "logits/chosen": 0.73046875, | |
| "logits/rejected": 0.5234375, | |
| "logps/chosen": -524.0, | |
| "logps/rejected": -728.0, | |
| "loss": 0.0387, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -10.6875, | |
| "rewards/margins": 9.5625, | |
| "rewards/rejected": -20.25, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 0.44280442804428044, | |
| "grad_norm": 43.99552374061499, | |
| "learning_rate": 3.1006160164271044e-07, | |
| "logits/chosen": 0.76171875, | |
| "logits/rejected": 0.8359375, | |
| "logps/chosen": -636.0, | |
| "logps/rejected": -760.0, | |
| "loss": 0.176, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -8.9375, | |
| "rewards/margins": 11.9375, | |
| "rewards/rejected": -20.875, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 0.4612546125461255, | |
| "grad_norm": 8.464204395396726, | |
| "learning_rate": 2.997946611909651e-07, | |
| "logits/chosen": 0.75390625, | |
| "logits/rejected": 0.86328125, | |
| "logps/chosen": -632.0, | |
| "logps/rejected": -704.0, | |
| "loss": 0.1227, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -9.0625, | |
| "rewards/margins": 12.25, | |
| "rewards/rejected": -21.375, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 0.4797047970479705, | |
| "grad_norm": 31.535660394992885, | |
| "learning_rate": 2.895277207392197e-07, | |
| "logits/chosen": 0.42578125, | |
| "logits/rejected": 0.6171875, | |
| "logps/chosen": -516.0, | |
| "logps/rejected": -704.0, | |
| "loss": 0.0264, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": -8.875, | |
| "rewards/margins": 11.0, | |
| "rewards/rejected": -19.875, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 0.4981549815498155, | |
| "grad_norm": 2.9983262496071896, | |
| "learning_rate": 2.792607802874743e-07, | |
| "logits/chosen": 0.40625, | |
| "logits/rejected": 0.35546875, | |
| "logps/chosen": -556.0, | |
| "logps/rejected": -644.0, | |
| "loss": 0.1141, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -8.5625, | |
| "rewards/margins": 11.3125, | |
| "rewards/rejected": -19.875, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 0.5166051660516605, | |
| "grad_norm": 23.55317172012457, | |
| "learning_rate": 2.6899383983572895e-07, | |
| "logits/chosen": 0.52734375, | |
| "logits/rejected": 0.59765625, | |
| "logps/chosen": -580.0, | |
| "logps/rejected": -872.0, | |
| "loss": 0.0199, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -9.25, | |
| "rewards/margins": 14.5, | |
| "rewards/rejected": -23.75, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 0.5350553505535055, | |
| "grad_norm": 103.51955740713792, | |
| "learning_rate": 2.5872689938398356e-07, | |
| "logits/chosen": 0.59765625, | |
| "logits/rejected": 0.5, | |
| "logps/chosen": -576.0, | |
| "logps/rejected": -752.0, | |
| "loss": 0.0535, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -11.25, | |
| "rewards/margins": 10.9375, | |
| "rewards/rejected": -22.25, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 0.5535055350553506, | |
| "grad_norm": 12.070322645461912, | |
| "learning_rate": 2.4845995893223816e-07, | |
| "logits/chosen": 0.78125, | |
| "logits/rejected": 0.52734375, | |
| "logps/chosen": -450.0, | |
| "logps/rejected": -696.0, | |
| "loss": 0.0148, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -9.0625, | |
| "rewards/margins": 12.25, | |
| "rewards/rejected": -21.25, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.5719557195571956, | |
| "grad_norm": 8.165485967165731, | |
| "learning_rate": 2.381930184804928e-07, | |
| "logits/chosen": 0.1787109375, | |
| "logits/rejected": 0.2099609375, | |
| "logps/chosen": -604.0, | |
| "logps/rejected": -728.0, | |
| "loss": 0.0809, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -12.0625, | |
| "rewards/margins": 10.375, | |
| "rewards/rejected": -22.375, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 0.5904059040590406, | |
| "grad_norm": 165.78369402101114, | |
| "learning_rate": 2.2792607802874742e-07, | |
| "logits/chosen": 0.96875, | |
| "logits/rejected": 0.86328125, | |
| "logps/chosen": -552.0, | |
| "logps/rejected": -672.0, | |
| "loss": 0.1083, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -11.0, | |
| "rewards/margins": 9.9375, | |
| "rewards/rejected": -21.0, | |
| "step": 320 | |
| }, | |
| { | |
| "epoch": 0.6088560885608856, | |
| "grad_norm": 1.3339261417313772, | |
| "learning_rate": 2.1765913757700204e-07, | |
| "logits/chosen": 0.50390625, | |
| "logits/rejected": 0.49609375, | |
| "logps/chosen": -556.0, | |
| "logps/rejected": -692.0, | |
| "loss": 0.0313, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -11.5, | |
| "rewards/margins": 13.125, | |
| "rewards/rejected": -24.625, | |
| "step": 330 | |
| }, | |
| { | |
| "epoch": 0.6273062730627307, | |
| "grad_norm": 15.585032380589984, | |
| "learning_rate": 2.0739219712525665e-07, | |
| "logits/chosen": 1.125, | |
| "logits/rejected": 0.9765625, | |
| "logps/chosen": -632.0, | |
| "logps/rejected": -656.0, | |
| "loss": 0.0868, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -11.25, | |
| "rewards/margins": 9.125, | |
| "rewards/rejected": -20.375, | |
| "step": 340 | |
| }, | |
| { | |
| "epoch": 0.6457564575645757, | |
| "grad_norm": 56.90922045086778, | |
| "learning_rate": 1.971252566735113e-07, | |
| "logits/chosen": 0.57421875, | |
| "logits/rejected": 0.494140625, | |
| "logps/chosen": -740.0, | |
| "logps/rejected": -808.0, | |
| "loss": 0.0303, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -10.6875, | |
| "rewards/margins": 15.0625, | |
| "rewards/rejected": -25.75, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 0.6642066420664207, | |
| "grad_norm": 209.39087027497493, | |
| "learning_rate": 1.868583162217659e-07, | |
| "logits/chosen": 0.859375, | |
| "logits/rejected": 0.7734375, | |
| "logps/chosen": -588.0, | |
| "logps/rejected": -640.0, | |
| "loss": 0.0846, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -12.875, | |
| "rewards/margins": 8.25, | |
| "rewards/rejected": -21.125, | |
| "step": 360 | |
| }, | |
| { | |
| "epoch": 0.6826568265682657, | |
| "grad_norm": 37.61950535221825, | |
| "learning_rate": 1.7659137577002053e-07, | |
| "logits/chosen": 0.828125, | |
| "logits/rejected": 0.53515625, | |
| "logps/chosen": -656.0, | |
| "logps/rejected": -772.0, | |
| "loss": 0.0841, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -10.625, | |
| "rewards/margins": 15.0, | |
| "rewards/rejected": -25.625, | |
| "step": 370 | |
| }, | |
| { | |
| "epoch": 0.7011070110701108, | |
| "grad_norm": 13.99532289592601, | |
| "learning_rate": 1.6632443531827514e-07, | |
| "logits/chosen": 0.640625, | |
| "logits/rejected": 0.625, | |
| "logps/chosen": -490.0, | |
| "logps/rejected": -808.0, | |
| "loss": 0.6414, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -10.125, | |
| "rewards/margins": 16.5, | |
| "rewards/rejected": -26.625, | |
| "step": 380 | |
| }, | |
| { | |
| "epoch": 0.7195571955719557, | |
| "grad_norm": 18.684565711511567, | |
| "learning_rate": 1.5605749486652976e-07, | |
| "logits/chosen": 0.380859375, | |
| "logits/rejected": 0.40625, | |
| "logps/chosen": -520.0, | |
| "logps/rejected": -712.0, | |
| "loss": 0.0142, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -10.5625, | |
| "rewards/margins": 13.25, | |
| "rewards/rejected": -23.875, | |
| "step": 390 | |
| }, | |
| { | |
| "epoch": 0.7380073800738007, | |
| "grad_norm": 7.359684787510465, | |
| "learning_rate": 1.457905544147844e-07, | |
| "logits/chosen": 0.63671875, | |
| "logits/rejected": 0.7265625, | |
| "logps/chosen": -592.0, | |
| "logps/rejected": -636.0, | |
| "loss": 0.053, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -8.75, | |
| "rewards/margins": 12.0, | |
| "rewards/rejected": -20.75, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 0.7564575645756457, | |
| "grad_norm": 1.0634520237548124, | |
| "learning_rate": 1.3552361396303902e-07, | |
| "logits/chosen": 1.203125, | |
| "logits/rejected": 1.1796875, | |
| "logps/chosen": -524.0, | |
| "logps/rejected": -632.0, | |
| "loss": 0.0473, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": -9.9375, | |
| "rewards/margins": 15.0, | |
| "rewards/rejected": -25.0, | |
| "step": 410 | |
| }, | |
| { | |
| "epoch": 0.7749077490774908, | |
| "grad_norm": 1.4375858852806764, | |
| "learning_rate": 1.2525667351129362e-07, | |
| "logits/chosen": 0.40625, | |
| "logits/rejected": 0.404296875, | |
| "logps/chosen": -704.0, | |
| "logps/rejected": -624.0, | |
| "loss": 0.1246, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -10.75, | |
| "rewards/margins": 11.125, | |
| "rewards/rejected": -21.875, | |
| "step": 420 | |
| }, | |
| { | |
| "epoch": 0.7933579335793358, | |
| "grad_norm": 5.4241732842447234, | |
| "learning_rate": 1.1498973305954825e-07, | |
| "logits/chosen": 0.46875, | |
| "logits/rejected": 0.5703125, | |
| "logps/chosen": -664.0, | |
| "logps/rejected": -600.0, | |
| "loss": 0.0351, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -10.8125, | |
| "rewards/margins": 11.375, | |
| "rewards/rejected": -22.125, | |
| "step": 430 | |
| }, | |
| { | |
| "epoch": 0.8118081180811808, | |
| "grad_norm": 26.64177636678718, | |
| "learning_rate": 1.0472279260780287e-07, | |
| "logits/chosen": 0.609375, | |
| "logits/rejected": 0.56640625, | |
| "logps/chosen": -620.0, | |
| "logps/rejected": -728.0, | |
| "loss": 0.0186, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -11.25, | |
| "rewards/margins": 12.625, | |
| "rewards/rejected": -23.875, | |
| "step": 440 | |
| }, | |
| { | |
| "epoch": 0.8302583025830258, | |
| "grad_norm": 81.60607018558989, | |
| "learning_rate": 9.445585215605748e-08, | |
| "logits/chosen": 0.62109375, | |
| "logits/rejected": 0.515625, | |
| "logps/chosen": -612.0, | |
| "logps/rejected": -732.0, | |
| "loss": 0.0274, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -12.625, | |
| "rewards/margins": 15.4375, | |
| "rewards/rejected": -28.0, | |
| "step": 450 | |
| }, | |
| { | |
| "epoch": 0.8487084870848709, | |
| "grad_norm": 1.080724529109667, | |
| "learning_rate": 8.418891170431211e-08, | |
| "logits/chosen": 0.78515625, | |
| "logits/rejected": 0.87890625, | |
| "logps/chosen": -680.0, | |
| "logps/rejected": -676.0, | |
| "loss": 0.0117, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -11.5, | |
| "rewards/margins": 15.1875, | |
| "rewards/rejected": -26.625, | |
| "step": 460 | |
| }, | |
| { | |
| "epoch": 0.8671586715867159, | |
| "grad_norm": 0.08637420796636854, | |
| "learning_rate": 7.392197125256673e-08, | |
| "logits/chosen": 0.419921875, | |
| "logits/rejected": 0.47265625, | |
| "logps/chosen": -588.0, | |
| "logps/rejected": -704.0, | |
| "loss": 0.0586, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -13.125, | |
| "rewards/margins": 14.9375, | |
| "rewards/rejected": -28.125, | |
| "step": 470 | |
| }, | |
| { | |
| "epoch": 0.8856088560885609, | |
| "grad_norm": 9.798255114201039, | |
| "learning_rate": 6.365503080082136e-08, | |
| "logits/chosen": 1.0234375, | |
| "logits/rejected": 0.59375, | |
| "logps/chosen": -470.0, | |
| "logps/rejected": -784.0, | |
| "loss": 0.0286, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -10.4375, | |
| "rewards/margins": 16.0, | |
| "rewards/rejected": -26.375, | |
| "step": 480 | |
| }, | |
| { | |
| "epoch": 0.9040590405904059, | |
| "grad_norm": 130.06407939428343, | |
| "learning_rate": 5.338809034907597e-08, | |
| "logits/chosen": 0.69140625, | |
| "logits/rejected": 0.345703125, | |
| "logps/chosen": -664.0, | |
| "logps/rejected": -672.0, | |
| "loss": 0.0588, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": -13.3125, | |
| "rewards/margins": 13.1875, | |
| "rewards/rejected": -26.5, | |
| "step": 490 | |
| }, | |
| { | |
| "epoch": 0.922509225092251, | |
| "grad_norm": 0.05379447574026675, | |
| "learning_rate": 4.3121149897330595e-08, | |
| "logits/chosen": 0.6796875, | |
| "logits/rejected": 0.353515625, | |
| "logps/chosen": -712.0, | |
| "logps/rejected": -880.0, | |
| "loss": 0.1362, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -12.3125, | |
| "rewards/margins": 14.5, | |
| "rewards/rejected": -26.75, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 0.940959409594096, | |
| "grad_norm": 134.79964147834983, | |
| "learning_rate": 3.285420944558522e-08, | |
| "logits/chosen": 0.404296875, | |
| "logits/rejected": 0.55859375, | |
| "logps/chosen": -564.0, | |
| "logps/rejected": -768.0, | |
| "loss": 0.0569, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": -12.875, | |
| "rewards/margins": 14.875, | |
| "rewards/rejected": -27.75, | |
| "step": 510 | |
| }, | |
| { | |
| "epoch": 0.959409594095941, | |
| "grad_norm": 18.208328849261832, | |
| "learning_rate": 2.2587268993839836e-08, | |
| "logits/chosen": 0.66796875, | |
| "logits/rejected": 0.43359375, | |
| "logps/chosen": -564.0, | |
| "logps/rejected": -836.0, | |
| "loss": 0.052, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -12.75, | |
| "rewards/margins": 15.0, | |
| "rewards/rejected": -27.75, | |
| "step": 520 | |
| }, | |
| { | |
| "epoch": 0.977859778597786, | |
| "grad_norm": 0.955662802081157, | |
| "learning_rate": 1.2320328542094456e-08, | |
| "logits/chosen": 0.427734375, | |
| "logits/rejected": 0.484375, | |
| "logps/chosen": -540.0, | |
| "logps/rejected": -700.0, | |
| "loss": 0.0037, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -11.9375, | |
| "rewards/margins": 15.9375, | |
| "rewards/rejected": -27.875, | |
| "step": 530 | |
| }, | |
| { | |
| "epoch": 0.996309963099631, | |
| "grad_norm": 0.3977726554857907, | |
| "learning_rate": 2.053388090349076e-09, | |
| "logits/chosen": 0.97265625, | |
| "logits/rejected": 0.828125, | |
| "logps/chosen": -548.0, | |
| "logps/rejected": -712.0, | |
| "loss": 0.0592, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -12.3125, | |
| "rewards/margins": 13.5625, | |
| "rewards/rejected": -25.875, | |
| "step": 540 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "eval_logits/chosen": 1.0078125, | |
| "eval_logits/rejected": 0.7734375, | |
| "eval_logps/chosen": -504.0, | |
| "eval_logps/rejected": -688.0, | |
| "eval_loss": 0.029836805537343025, | |
| "eval_rewards/accuracies": 1.0, | |
| "eval_rewards/chosen": -12.75, | |
| "eval_rewards/margins": 16.0, | |
| "eval_rewards/rejected": -28.75, | |
| "eval_runtime": 7.0041, | |
| "eval_samples_per_second": 14.277, | |
| "eval_steps_per_second": 1.856, | |
| "step": 542 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "step": 542, | |
| "total_flos": 0.0, | |
| "train_loss": 0.13226419520938967, | |
| "train_runtime": 912.7927, | |
| "train_samples_per_second": 4.746, | |
| "train_steps_per_second": 0.594 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 542, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 500, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 2, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |