| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 2.0, | |
| "eval_steps": 500, | |
| "global_step": 712, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.0028089887640449437, | |
| "grad_norm": 1.1993484497070312, | |
| "learning_rate": 6.9444444444444435e-09, | |
| "logits/chosen": -3.220703125, | |
| "logits/rejected": -3.1796875, | |
| "logps/chosen": -43.75, | |
| "logps/rejected": -42.78125, | |
| "loss": 0.6914, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "epoch": 0.028089887640449437, | |
| "grad_norm": 0.7986037135124207, | |
| "learning_rate": 6.944444444444444e-08, | |
| "logits/chosen": -3.2486979961395264, | |
| "logits/rejected": -3.2052950859069824, | |
| "logps/chosen": -42.34375, | |
| "logps/rejected": -41.01215362548828, | |
| "loss": 0.6921, | |
| "rewards/accuracies": 0.2161458283662796, | |
| "rewards/chosen": 0.00020620558643713593, | |
| "rewards/margins": -0.00015253490710165352, | |
| "rewards/rejected": 0.0003587139945011586, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.056179775280898875, | |
| "grad_norm": 0.6845986247062683, | |
| "learning_rate": 1.3888888888888888e-07, | |
| "logits/chosen": -3.244140625, | |
| "logits/rejected": -3.219531297683716, | |
| "logps/chosen": -41.37812423706055, | |
| "logps/rejected": -40.42656326293945, | |
| "loss": 0.6907, | |
| "rewards/accuracies": 0.3460937440395355, | |
| "rewards/chosen": 0.006803750991821289, | |
| "rewards/margins": 0.003222918603569269, | |
| "rewards/rejected": 0.003580451011657715, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.08426966292134831, | |
| "grad_norm": 0.5183084607124329, | |
| "learning_rate": 2.0833333333333333e-07, | |
| "logits/chosen": -3.263671875, | |
| "logits/rejected": -3.2261719703674316, | |
| "logps/chosen": -41.142189025878906, | |
| "logps/rejected": -40.00312423706055, | |
| "loss": 0.6862, | |
| "rewards/accuracies": 0.44140625, | |
| "rewards/chosen": 0.02759246900677681, | |
| "rewards/margins": 0.013646435923874378, | |
| "rewards/rejected": 0.013950538821518421, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.11235955056179775, | |
| "grad_norm": 0.5848017334938049, | |
| "learning_rate": 2.7777777777777776e-07, | |
| "logits/chosen": -3.286328077316284, | |
| "logits/rejected": -3.229296922683716, | |
| "logps/chosen": -41.439064025878906, | |
| "logps/rejected": -41.0, | |
| "loss": 0.6745, | |
| "rewards/accuracies": 0.5546875, | |
| "rewards/chosen": 0.039928339421749115, | |
| "rewards/margins": 0.038716744631528854, | |
| "rewards/rejected": 0.001224255538545549, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.1404494382022472, | |
| "grad_norm": 0.7922030091285706, | |
| "learning_rate": 3.472222222222222e-07, | |
| "logits/chosen": -3.2728514671325684, | |
| "logits/rejected": NaN, | |
| "logps/chosen": -41.71406173706055, | |
| "logps/rejected": -41.890625, | |
| "loss": 0.6535, | |
| "rewards/accuracies": 0.645312488079071, | |
| "rewards/chosen": 0.03761863708496094, | |
| "rewards/margins": 0.08368835598230362, | |
| "rewards/rejected": -0.04610452800989151, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.16853932584269662, | |
| "grad_norm": 1.3905400037765503, | |
| "learning_rate": 4.1666666666666667e-07, | |
| "logits/chosen": -3.204296827316284, | |
| "logits/rejected": -3.154296875, | |
| "logps/chosen": -44.493751525878906, | |
| "logps/rejected": -48.446876525878906, | |
| "loss": 0.5839, | |
| "rewards/accuracies": 0.692187488079071, | |
| "rewards/chosen": -0.11696071922779083, | |
| "rewards/margins": 0.25516968965530396, | |
| "rewards/rejected": -0.3720703125, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.19662921348314608, | |
| "grad_norm": 0.8105548024177551, | |
| "learning_rate": 4.861111111111111e-07, | |
| "logits/chosen": -3.23828125, | |
| "logits/rejected": -3.130859375, | |
| "logps/chosen": -47.71875, | |
| "logps/rejected": -59.140625, | |
| "loss": 0.4754, | |
| "rewards/accuracies": 0.7093750238418579, | |
| "rewards/chosen": -0.2639709413051605, | |
| "rewards/margins": 0.6321045160293579, | |
| "rewards/rejected": -0.895751953125, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.2247191011235955, | |
| "grad_norm": 0.6645036339759827, | |
| "learning_rate": 4.998072590601808e-07, | |
| "logits/chosen": -3.2529296875, | |
| "logits/rejected": NaN, | |
| "logps/chosen": -43.765625, | |
| "logps/rejected": -72.40937805175781, | |
| "loss": 0.3394, | |
| "rewards/accuracies": 0.73828125, | |
| "rewards/chosen": -0.08822021633386612, | |
| "rewards/margins": 1.5029785633087158, | |
| "rewards/rejected": -1.591284155845642, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.25280898876404495, | |
| "grad_norm": 0.436211496591568, | |
| "learning_rate": 4.990247583129217e-07, | |
| "logits/chosen": -3.268749952316284, | |
| "logits/rejected": -2.9375, | |
| "logps/chosen": -46.046875, | |
| "logps/rejected": -89.46875, | |
| "loss": 0.2745, | |
| "rewards/accuracies": 0.7476562261581421, | |
| "rewards/chosen": -0.22807636857032776, | |
| "rewards/margins": 2.234375, | |
| "rewards/rejected": -2.463183641433716, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.2808988764044944, | |
| "grad_norm": 0.43823757767677307, | |
| "learning_rate": 4.976423351108942e-07, | |
| "logits/chosen": -3.2183594703674316, | |
| "logits/rejected": NaN, | |
| "logps/chosen": -49.96875, | |
| "logps/rejected": -105.32499694824219, | |
| "loss": 0.2356, | |
| "rewards/accuracies": 0.75390625, | |
| "rewards/chosen": -0.3900146484375, | |
| "rewards/margins": 2.8427734375, | |
| "rewards/rejected": -3.232421875, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.3089887640449438, | |
| "grad_norm": 0.6719679832458496, | |
| "learning_rate": 4.95663319832678e-07, | |
| "logits/chosen": -3.2197265625, | |
| "logits/rejected": -2.762890577316284, | |
| "logps/chosen": -46.25312423706055, | |
| "logps/rejected": -114.41874694824219, | |
| "loss": 0.236, | |
| "rewards/accuracies": 0.7523437738418579, | |
| "rewards/chosen": -0.22943687438964844, | |
| "rewards/margins": 3.4632811546325684, | |
| "rewards/rejected": -3.6929688453674316, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.33707865168539325, | |
| "grad_norm": 0.3029685914516449, | |
| "learning_rate": 4.930924800994191e-07, | |
| "logits/chosen": -3.2417969703674316, | |
| "logits/rejected": -2.752734422683716, | |
| "logps/chosen": -45.203125, | |
| "logps/rejected": -123.83125305175781, | |
| "loss": 0.1955, | |
| "rewards/accuracies": 0.77734375, | |
| "rewards/chosen": -0.130183607339859, | |
| "rewards/margins": 4.001757621765137, | |
| "rewards/rejected": -4.1328125, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.3651685393258427, | |
| "grad_norm": 0.5936453938484192, | |
| "learning_rate": 4.899360092892144e-07, | |
| "logits/chosen": -3.2191405296325684, | |
| "logits/rejected": -2.7181639671325684, | |
| "logps/chosen": -46.69843673706055, | |
| "logps/rejected": -128.28750610351562, | |
| "loss": 0.2039, | |
| "rewards/accuracies": 0.7679687738418579, | |
| "rewards/chosen": -0.2604345381259918, | |
| "rewards/margins": 4.139843940734863, | |
| "rewards/rejected": -4.399609565734863, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.39325842696629215, | |
| "grad_norm": 0.4349266588687897, | |
| "learning_rate": 4.862015116167195e-07, | |
| "logits/chosen": -3.243945360183716, | |
| "logits/rejected": NaN, | |
| "logps/chosen": -45.373435974121094, | |
| "logps/rejected": -133.71249389648438, | |
| "loss": 0.1869, | |
| "rewards/accuracies": 0.7789062261581421, | |
| "rewards/chosen": -0.10051727294921875, | |
| "rewards/margins": 4.529296875, | |
| "rewards/rejected": -4.631054878234863, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.42134831460674155, | |
| "grad_norm": 0.32329249382019043, | |
| "learning_rate": 4.81897983813931e-07, | |
| "logits/chosen": -3.2386717796325684, | |
| "logits/rejected": -2.702929735183716, | |
| "logps/chosen": -46.21562576293945, | |
| "logps/rejected": -138.9375, | |
| "loss": 0.1833, | |
| "rewards/accuracies": 0.780468761920929, | |
| "rewards/chosen": -0.20498999953269958, | |
| "rewards/margins": 4.690625190734863, | |
| "rewards/rejected": -4.895898342132568, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.449438202247191, | |
| "grad_norm": 0.24011513590812683, | |
| "learning_rate": 4.770357934562704e-07, | |
| "logits/chosen": -3.281445264816284, | |
| "logits/rejected": NaN, | |
| "logps/chosen": -41.82500076293945, | |
| "logps/rejected": -132.2624969482422, | |
| "loss": 0.195, | |
| "rewards/accuracies": 0.765625, | |
| "rewards/chosen": 0.00346546177752316, | |
| "rewards/margins": 4.6083984375, | |
| "rewards/rejected": -4.605273246765137, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 0.47752808988764045, | |
| "grad_norm": 0.5602438449859619, | |
| "learning_rate": 4.716266539861866e-07, | |
| "logits/chosen": -3.237499952316284, | |
| "logits/rejected": -2.6552734375, | |
| "logps/chosen": -47.9921875, | |
| "logps/rejected": -144.77499389648438, | |
| "loss": 0.1834, | |
| "rewards/accuracies": 0.7757812738418579, | |
| "rewards/chosen": -0.31496095657348633, | |
| "rewards/margins": 4.906640529632568, | |
| "rewards/rejected": -5.219336032867432, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 0.5056179775280899, | |
| "grad_norm": 0.5013967156410217, | |
| "learning_rate": 4.6568359649444796e-07, | |
| "logits/chosen": -3.225781202316284, | |
| "logits/rejected": -2.6431641578674316, | |
| "logps/chosen": -47.4140625, | |
| "logps/rejected": -147.6062469482422, | |
| "loss": 0.1777, | |
| "rewards/accuracies": 0.7835937738418579, | |
| "rewards/chosen": -0.26249465346336365, | |
| "rewards/margins": 5.070898532867432, | |
| "rewards/rejected": -5.333398342132568, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 0.5337078651685393, | |
| "grad_norm": 0.3109186887741089, | |
| "learning_rate": 4.592209383271023e-07, | |
| "logits/chosen": -3.2769532203674316, | |
| "logits/rejected": -2.708203077316284, | |
| "logps/chosen": -40.32500076293945, | |
| "logps/rejected": -146.55624389648438, | |
| "loss": 0.1691, | |
| "rewards/accuracies": 0.7945312261581421, | |
| "rewards/chosen": 0.10864410549402237, | |
| "rewards/margins": 5.372851371765137, | |
| "rewards/rejected": -5.262499809265137, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 0.5617977528089888, | |
| "grad_norm": 0.329673707485199, | |
| "learning_rate": 4.5225424859373684e-07, | |
| "logits/chosen": -3.244921922683716, | |
| "logits/rejected": -2.625195264816284, | |
| "logps/chosen": -46.1875, | |
| "logps/rejected": -152.55624389648438, | |
| "loss": 0.1722, | |
| "rewards/accuracies": 0.784375011920929, | |
| "rewards/chosen": -0.20301513373851776, | |
| "rewards/margins": 5.3759765625, | |
| "rewards/rejected": -5.578711032867432, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.5898876404494382, | |
| "grad_norm": 0.6182069182395935, | |
| "learning_rate": 4.448003106601291e-07, | |
| "logits/chosen": -3.2255859375, | |
| "logits/rejected": NaN, | |
| "logps/chosen": -45.904685974121094, | |
| "logps/rejected": -153.14999389648438, | |
| "loss": 0.1818, | |
| "rewards/accuracies": 0.7789062261581421, | |
| "rewards/chosen": -0.2197813093662262, | |
| "rewards/margins": 5.441601753234863, | |
| "rewards/rejected": -5.661913871765137, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 0.6179775280898876, | |
| "grad_norm": 0.10240374505519867, | |
| "learning_rate": 4.3687708171564917e-07, | |
| "logits/chosen": -3.2201170921325684, | |
| "logits/rejected": NaN, | |
| "logps/chosen": -46.890625, | |
| "logps/rejected": -154.24374389648438, | |
| "loss": 0.18, | |
| "rewards/accuracies": 0.76953125, | |
| "rewards/chosen": -0.2859039306640625, | |
| "rewards/margins": 5.441992282867432, | |
| "rewards/rejected": -5.730078220367432, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 0.6460674157303371, | |
| "grad_norm": 0.5484092235565186, | |
| "learning_rate": 4.2850364951281705e-07, | |
| "logits/chosen": -3.2359375953674316, | |
| "logits/rejected": -2.595507860183716, | |
| "logps/chosen": -44.52656173706055, | |
| "logps/rejected": -156.5187530517578, | |
| "loss": 0.1763, | |
| "rewards/accuracies": 0.785937488079071, | |
| "rewards/chosen": -0.10360870510339737, | |
| "rewards/margins": 5.668359279632568, | |
| "rewards/rejected": -5.7724609375, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 0.6741573033707865, | |
| "grad_norm": 0.33201783895492554, | |
| "learning_rate": 4.1970018638323547e-07, | |
| "logits/chosen": -3.2476563453674316, | |
| "logits/rejected": -2.6099610328674316, | |
| "logps/chosen": -41.240623474121094, | |
| "logps/rejected": -152.3249969482422, | |
| "loss": 0.1836, | |
| "rewards/accuracies": 0.778124988079071, | |
| "rewards/chosen": 0.05380706861615181, | |
| "rewards/margins": 5.612500190734863, | |
| "rewards/rejected": -5.559960842132568, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 0.702247191011236, | |
| "grad_norm": 0.3225279450416565, | |
| "learning_rate": 4.1048790064067573e-07, | |
| "logits/chosen": -3.2152342796325684, | |
| "logits/rejected": NaN, | |
| "logps/chosen": -42.44062423706055, | |
| "logps/rejected": -152.5437469482422, | |
| "loss": 0.1874, | |
| "rewards/accuracies": 0.7679687738418579, | |
| "rewards/chosen": -0.019408416002988815, | |
| "rewards/margins": 5.590234279632568, | |
| "rewards/rejected": -5.608788967132568, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 0.7303370786516854, | |
| "grad_norm": 0.5013627409934998, | |
| "learning_rate": 4.0088898548839285e-07, | |
| "logits/chosen": -3.1996092796325684, | |
| "logits/rejected": -2.558398485183716, | |
| "logps/chosen": -46.775001525878906, | |
| "logps/rejected": -157.64999389648438, | |
| "loss": 0.1809, | |
| "rewards/accuracies": 0.78125, | |
| "rewards/chosen": -0.24601340293884277, | |
| "rewards/margins": 5.5927734375, | |
| "rewards/rejected": -5.837695121765137, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 0.7584269662921348, | |
| "grad_norm": 0.2542474567890167, | |
| "learning_rate": 3.9092656555375414e-07, | |
| "logits/chosen": -3.184765577316284, | |
| "logits/rejected": -2.549999952316284, | |
| "logps/chosen": -48.16718673706055, | |
| "logps/rejected": -155.3937530517578, | |
| "loss": 0.1927, | |
| "rewards/accuracies": 0.7593749761581421, | |
| "rewards/chosen": -0.3481277525424957, | |
| "rewards/margins": 5.426171779632568, | |
| "rewards/rejected": -5.775000095367432, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 0.7865168539325843, | |
| "grad_norm": 1.486909031867981, | |
| "learning_rate": 3.806246411789872e-07, | |
| "logits/chosen": -3.206835985183716, | |
| "logits/rejected": -2.568554639816284, | |
| "logps/chosen": -46.279685974121094, | |
| "logps/rejected": -161.4499969482422, | |
| "loss": 0.1695, | |
| "rewards/accuracies": 0.796875, | |
| "rewards/chosen": -0.17198029160499573, | |
| "rewards/margins": 5.812304496765137, | |
| "rewards/rejected": -5.983984470367432, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 0.8146067415730337, | |
| "grad_norm": 0.24136996269226074, | |
| "learning_rate": 3.700080306022528e-07, | |
| "logits/chosen": -3.238085985183716, | |
| "logits/rejected": -2.606250047683716, | |
| "logps/chosen": -42.40312576293945, | |
| "logps/rejected": -155.80624389648438, | |
| "loss": 0.1873, | |
| "rewards/accuracies": 0.76953125, | |
| "rewards/chosen": -0.016921233385801315, | |
| "rewards/margins": 5.745703220367432, | |
| "rewards/rejected": -5.7607421875, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 0.8426966292134831, | |
| "grad_norm": 0.3395313620567322, | |
| "learning_rate": 3.5910231016833546e-07, | |
| "logits/chosen": -3.2119140625, | |
| "logits/rejected": NaN, | |
| "logps/chosen": -45.810935974121094, | |
| "logps/rejected": -161.1687469482422, | |
| "loss": 0.174, | |
| "rewards/accuracies": 0.78125, | |
| "rewards/chosen": -0.18533477187156677, | |
| "rewards/margins": 5.834374904632568, | |
| "rewards/rejected": -6.019140720367432, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.8707865168539326, | |
| "grad_norm": 0.14939536154270172, | |
| "learning_rate": 3.4793375271298895e-07, | |
| "logits/chosen": -3.1988282203674316, | |
| "logits/rejected": -2.597851514816284, | |
| "logps/chosen": -44.76874923706055, | |
| "logps/rejected": -163.0437469482422, | |
| "loss": 0.1656, | |
| "rewards/accuracies": 0.7867187261581421, | |
| "rewards/chosen": -0.11381302028894424, | |
| "rewards/margins": 5.981835842132568, | |
| "rewards/rejected": -6.096875190734863, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 0.898876404494382, | |
| "grad_norm": 0.6819726228713989, | |
| "learning_rate": 3.3652926426937325e-07, | |
| "logits/chosen": -3.246875047683716, | |
| "logits/rejected": -2.609179735183716, | |
| "logps/chosen": -41.875, | |
| "logps/rejected": -162.28125, | |
| "loss": 0.1824, | |
| "rewards/accuracies": 0.7828124761581421, | |
| "rewards/chosen": 0.06156463548541069, | |
| "rewards/margins": 6.103515625, | |
| "rewards/rejected": -6.043359279632568, | |
| "step": 320 | |
| }, | |
| { | |
| "epoch": 0.9269662921348315, | |
| "grad_norm": 0.10508494079113007, | |
| "learning_rate": 3.249163192490642e-07, | |
| "logits/chosen": -3.2544922828674316, | |
| "logits/rejected": -2.624804735183716, | |
| "logps/chosen": -41.25, | |
| "logps/rejected": -159.97500610351562, | |
| "loss": 0.1816, | |
| "rewards/accuracies": 0.7734375, | |
| "rewards/chosen": 0.021668624132871628, | |
| "rewards/margins": 5.989648342132568, | |
| "rewards/rejected": -5.967968940734863, | |
| "step": 330 | |
| }, | |
| { | |
| "epoch": 0.9550561797752809, | |
| "grad_norm": 0.2005680650472641, | |
| "learning_rate": 3.1312289425378944e-07, | |
| "logits/chosen": -3.2759766578674316, | |
| "logits/rejected": -2.6624999046325684, | |
| "logps/chosen": -39.45781326293945, | |
| "logps/rejected": -157.4375, | |
| "loss": 0.1757, | |
| "rewards/accuracies": 0.78125, | |
| "rewards/chosen": 0.14164963364601135, | |
| "rewards/margins": 5.972851753234863, | |
| "rewards/rejected": -5.831250190734863, | |
| "step": 340 | |
| }, | |
| { | |
| "epoch": 0.9831460674157303, | |
| "grad_norm": 0.2797287404537201, | |
| "learning_rate": 3.011774006773449e-07, | |
| "logits/chosen": -3.282031297683716, | |
| "logits/rejected": NaN, | |
| "logps/chosen": -37.98906326293945, | |
| "logps/rejected": -157.4375, | |
| "loss": 0.1713, | |
| "rewards/accuracies": 0.7757812738418579, | |
| "rewards/chosen": 0.19436034560203552, | |
| "rewards/margins": 6.061718940734863, | |
| "rewards/rejected": -5.866406440734863, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 1.0112359550561798, | |
| "grad_norm": 0.1386144906282425, | |
| "learning_rate": 2.8910861626005773e-07, | |
| "logits/chosen": -3.2330079078674316, | |
| "logits/rejected": -2.6128907203674316, | |
| "logps/chosen": -42.69062423706055, | |
| "logps/rejected": -162.75625610351562, | |
| "loss": 0.1689, | |
| "rewards/accuracies": 0.7942708730697632, | |
| "rewards/chosen": -0.019468307495117188, | |
| "rewards/margins": 6.076367378234863, | |
| "rewards/rejected": -6.09765625, | |
| "step": 360 | |
| }, | |
| { | |
| "epoch": 1.0393258426966292, | |
| "grad_norm": 0.31265443563461304, | |
| "learning_rate": 2.7694561576068983e-07, | |
| "logits/chosen": -3.234375, | |
| "logits/rejected": -2.5785155296325684, | |
| "logps/chosen": -41.896873474121094, | |
| "logps/rejected": -160.8125, | |
| "loss": 0.1713, | |
| "rewards/accuracies": 0.7867187261581421, | |
| "rewards/chosen": -0.01103897113353014, | |
| "rewards/margins": 6.00390625, | |
| "rewards/rejected": -6.014843940734863, | |
| "step": 370 | |
| }, | |
| { | |
| "epoch": 1.0674157303370786, | |
| "grad_norm": 0.3972986042499542, | |
| "learning_rate": 2.647177009127972e-07, | |
| "logits/chosen": -3.2544922828674316, | |
| "logits/rejected": -2.5933594703674316, | |
| "logps/chosen": -39.5703125, | |
| "logps/rejected": -160.85000610351562, | |
| "loss": 0.1793, | |
| "rewards/accuracies": 0.768750011920929, | |
| "rewards/chosen": 0.09815521538257599, | |
| "rewards/margins": 6.125781059265137, | |
| "rewards/rejected": -6.029492378234863, | |
| "step": 380 | |
| }, | |
| { | |
| "epoch": 1.095505617977528, | |
| "grad_norm": 0.12721529603004456, | |
| "learning_rate": 2.524543298342874e-07, | |
| "logits/chosen": -3.262890577316284, | |
| "logits/rejected": -2.592578172683716, | |
| "logps/chosen": -40.4765625, | |
| "logps/rejected": -161.91250610351562, | |
| "loss": 0.1743, | |
| "rewards/accuracies": 0.77734375, | |
| "rewards/chosen": 0.05222644656896591, | |
| "rewards/margins": 6.142578125, | |
| "rewards/rejected": -6.091406345367432, | |
| "step": 390 | |
| }, | |
| { | |
| "epoch": 1.1235955056179776, | |
| "grad_norm": 0.09501045197248459, | |
| "learning_rate": 2.401850460602329e-07, | |
| "logits/chosen": -3.2220702171325684, | |
| "logits/rejected": -2.559765577316284, | |
| "logps/chosen": -44.423439025878906, | |
| "logps/rejected": -167.08749389648438, | |
| "loss": 0.1718, | |
| "rewards/accuracies": 0.780468761920929, | |
| "rewards/chosen": -0.09567908942699432, | |
| "rewards/margins": 6.20703125, | |
| "rewards/rejected": -6.302734375, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 1.151685393258427, | |
| "grad_norm": 0.29580923914909363, | |
| "learning_rate": 2.2793940736990766e-07, | |
| "logits/chosen": -3.2007813453674316, | |
| "logits/rejected": NaN, | |
| "logps/chosen": -51.12968826293945, | |
| "logps/rejected": -170.96875, | |
| "loss": 0.172, | |
| "rewards/accuracies": 0.780468761920929, | |
| "rewards/chosen": -0.42823487520217896, | |
| "rewards/margins": 6.079297065734863, | |
| "rewards/rejected": -6.508593559265137, | |
| "step": 410 | |
| }, | |
| { | |
| "epoch": 1.1797752808988764, | |
| "grad_norm": 0.2765229046344757, | |
| "learning_rate": 2.1574691457950803e-07, | |
| "logits/chosen": -3.1839842796325684, | |
| "logits/rejected": NaN, | |
| "logps/chosen": -50.9609375, | |
| "logps/rejected": -170.78125, | |
| "loss": 0.1704, | |
| "rewards/accuracies": 0.7734375, | |
| "rewards/chosen": -0.46327972412109375, | |
| "rewards/margins": 6.071484565734863, | |
| "rewards/rejected": -6.534375190734863, | |
| "step": 420 | |
| }, | |
| { | |
| "epoch": 1.2078651685393258, | |
| "grad_norm": 0.15300235152244568, | |
| "learning_rate": 2.036369404721023e-07, | |
| "logits/chosen": -3.2113280296325684, | |
| "logits/rejected": -2.531054735183716, | |
| "logps/chosen": -46.8828125, | |
| "logps/rejected": -173.1374969482422, | |
| "loss": 0.1599, | |
| "rewards/accuracies": 0.79296875, | |
| "rewards/chosen": -0.21221771836280823, | |
| "rewards/margins": 6.374609470367432, | |
| "rewards/rejected": -6.587109565734863, | |
| "step": 430 | |
| }, | |
| { | |
| "epoch": 1.2359550561797752, | |
| "grad_norm": 0.6692606806755066, | |
| "learning_rate": 1.9163865903602372e-07, | |
| "logits/chosen": -3.253124952316284, | |
| "logits/rejected": -2.5308594703674316, | |
| "logps/chosen": -42.287498474121094, | |
| "logps/rejected": -169.625, | |
| "loss": 0.1762, | |
| "rewards/accuracies": 0.776562511920929, | |
| "rewards/chosen": -0.029553985223174095, | |
| "rewards/margins": 6.40625, | |
| "rewards/rejected": -6.434765815734863, | |
| "step": 440 | |
| }, | |
| { | |
| "epoch": 1.2640449438202248, | |
| "grad_norm": 0.2745380103588104, | |
| "learning_rate": 1.7978097518217702e-07, | |
| "logits/chosen": -3.237499952316284, | |
| "logits/rejected": -2.541015625, | |
| "logps/chosen": -42.74687576293945, | |
| "logps/rejected": -168.78750610351562, | |
| "loss": 0.1715, | |
| "rewards/accuracies": 0.774218738079071, | |
| "rewards/chosen": -0.021244239062070847, | |
| "rewards/margins": 6.392187595367432, | |
| "rewards/rejected": -6.409375190734863, | |
| "step": 450 | |
| }, | |
| { | |
| "epoch": 1.2921348314606742, | |
| "grad_norm": 0.3858237564563751, | |
| "learning_rate": 1.6809245510957666e-07, | |
| "logits/chosen": -3.235546827316284, | |
| "logits/rejected": NaN, | |
| "logps/chosen": -44.904685974121094, | |
| "logps/rejected": -172.96249389648438, | |
| "loss": 0.1663, | |
| "rewards/accuracies": 0.7875000238418579, | |
| "rewards/chosen": -0.10267486423254013, | |
| "rewards/margins": 6.484375, | |
| "rewards/rejected": -6.587109565734863, | |
| "step": 460 | |
| }, | |
| { | |
| "epoch": 1.3202247191011236, | |
| "grad_norm": 0.25496408343315125, | |
| "learning_rate": 1.5660125748687093e-07, | |
| "logits/chosen": -3.224414110183716, | |
| "logits/rejected": NaN, | |
| "logps/chosen": -46.64374923706055, | |
| "logps/rejected": -171.1374969482422, | |
| "loss": 0.1703, | |
| "rewards/accuracies": 0.780468761920929, | |
| "rewards/chosen": -0.2242431640625, | |
| "rewards/margins": 6.318749904632568, | |
| "rewards/rejected": -6.542187690734863, | |
| "step": 470 | |
| }, | |
| { | |
| "epoch": 1.348314606741573, | |
| "grad_norm": 0.3984578251838684, | |
| "learning_rate": 1.4533506561564305e-07, | |
| "logits/chosen": -3.224609375, | |
| "logits/rejected": -2.529101610183716, | |
| "logps/chosen": -47.29375076293945, | |
| "logps/rejected": -173.3625030517578, | |
| "loss": 0.1669, | |
| "rewards/accuracies": 0.780468761920929, | |
| "rewards/chosen": -0.2605232298374176, | |
| "rewards/margins": 6.368359565734863, | |
| "rewards/rejected": -6.627734184265137, | |
| "step": 480 | |
| }, | |
| { | |
| "epoch": 1.3764044943820224, | |
| "grad_norm": 0.39370596408843994, | |
| "learning_rate": 1.343210207389125e-07, | |
| "logits/chosen": -3.203320264816284, | |
| "logits/rejected": -2.5302734375, | |
| "logps/chosen": -48.0859375, | |
| "logps/rejected": -172.93124389648438, | |
| "loss": 0.1747, | |
| "rewards/accuracies": 0.7757812738418579, | |
| "rewards/chosen": -0.27991026639938354, | |
| "rewards/margins": 6.329297065734863, | |
| "rewards/rejected": -6.610156059265137, | |
| "step": 490 | |
| }, | |
| { | |
| "epoch": 1.404494382022472, | |
| "grad_norm": 0.2013687640428543, | |
| "learning_rate": 1.2358565665550387e-07, | |
| "logits/chosen": -3.244335889816284, | |
| "logits/rejected": NaN, | |
| "logps/chosen": -41.96875, | |
| "logps/rejected": -176.03750610351562, | |
| "loss": 0.1557, | |
| "rewards/accuracies": 0.793749988079071, | |
| "rewards/chosen": 0.01244263630360365, | |
| "rewards/margins": 6.784765720367432, | |
| "rewards/rejected": -6.769921779632568, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 1.4325842696629214, | |
| "grad_norm": 0.24023930728435516, | |
| "learning_rate": 1.1315483579780094e-07, | |
| "logits/chosen": -3.2484374046325684, | |
| "logits/rejected": -2.5347657203674316, | |
| "logps/chosen": -42.046875, | |
| "logps/rejected": -175.8312530517578, | |
| "loss": 0.1643, | |
| "rewards/accuracies": 0.788281261920929, | |
| "rewards/chosen": 0.040112875401973724, | |
| "rewards/margins": 6.779296875, | |
| "rewards/rejected": -6.739453315734863, | |
| "step": 510 | |
| }, | |
| { | |
| "epoch": 1.4606741573033708, | |
| "grad_norm": 0.5066425800323486, | |
| "learning_rate": 1.0305368692688174e-07, | |
| "logits/chosen": -3.240429639816284, | |
| "logits/rejected": -2.522656202316284, | |
| "logps/chosen": -43.00468826293945, | |
| "logps/rejected": -173.8625030517578, | |
| "loss": 0.1791, | |
| "rewards/accuracies": 0.778124988079071, | |
| "rewards/chosen": -0.03853149339556694, | |
| "rewards/margins": 6.608788967132568, | |
| "rewards/rejected": -6.646874904632568, | |
| "step": 520 | |
| }, | |
| { | |
| "epoch": 1.4887640449438202, | |
| "grad_norm": 0.10324753075838089, | |
| "learning_rate": 9.330654459513266e-08, | |
| "logits/chosen": -3.233203172683716, | |
| "logits/rejected": -2.5269532203674316, | |
| "logps/chosen": -41.953125, | |
| "logps/rejected": -172.06875610351562, | |
| "loss": 0.1683, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -0.02672729454934597, | |
| "rewards/margins": 6.571875095367432, | |
| "rewards/rejected": -6.598437309265137, | |
| "step": 530 | |
| }, | |
| { | |
| "epoch": 1.5168539325842696, | |
| "grad_norm": 0.06884948909282684, | |
| "learning_rate": 8.393689052217964e-08, | |
| "logits/chosen": -3.2310547828674316, | |
| "logits/rejected": -2.5240235328674316, | |
| "logps/chosen": -41.89531326293945, | |
| "logps/rejected": -170.4499969482422, | |
| "loss": 0.1774, | |
| "rewards/accuracies": 0.770312488079071, | |
| "rewards/chosen": -0.03569946438074112, | |
| "rewards/margins": 6.483984470367432, | |
| "rewards/rejected": -6.521093845367432, | |
| "step": 540 | |
| }, | |
| { | |
| "epoch": 1.5449438202247192, | |
| "grad_norm": 0.2964838445186615, | |
| "learning_rate": 7.49672970253691e-08, | |
| "logits/chosen": -3.2562499046325684, | |
| "logits/rejected": -2.5296874046325684, | |
| "logps/chosen": -40.365623474121094, | |
| "logps/rejected": -173.3625030517578, | |
| "loss": 0.1644, | |
| "rewards/accuracies": 0.78515625, | |
| "rewards/chosen": 0.06430435180664062, | |
| "rewards/margins": 6.696484565734863, | |
| "rewards/rejected": -6.630078315734863, | |
| "step": 550 | |
| }, | |
| { | |
| "epoch": 1.5730337078651684, | |
| "grad_norm": 0.16073837876319885, | |
| "learning_rate": 6.641937264107867e-08, | |
| "logits/chosen": -3.2535157203674316, | |
| "logits/rejected": -2.5425782203674316, | |
| "logps/chosen": -39.43281173706055, | |
| "logps/rejected": -167.91250610351562, | |
| "loss": 0.1788, | |
| "rewards/accuracies": 0.7671874761581421, | |
| "rewards/chosen": 0.08403320610523224, | |
| "rewards/margins": 6.482421875, | |
| "rewards/rejected": -6.396874904632568, | |
| "step": 560 | |
| }, | |
| { | |
| "epoch": 1.601123595505618, | |
| "grad_norm": 0.27828249335289, | |
| "learning_rate": 5.831371006785962e-08, | |
| "logits/chosen": -3.2699217796325684, | |
| "logits/rejected": -2.5228514671325684, | |
| "logps/chosen": -39.87187576293945, | |
| "logps/rejected": -171.1437530517578, | |
| "loss": 0.175, | |
| "rewards/accuracies": 0.772656261920929, | |
| "rewards/chosen": 0.07085514068603516, | |
| "rewards/margins": 6.617578029632568, | |
| "rewards/rejected": -6.548047065734863, | |
| "step": 570 | |
| }, | |
| { | |
| "epoch": 1.6292134831460674, | |
| "grad_norm": 0.386394202709198, | |
| "learning_rate": 5.066983655682325e-08, | |
| "logits/chosen": -3.2455077171325684, | |
| "logits/rejected": -2.546093702316284, | |
| "logps/chosen": -40.72968673706055, | |
| "logps/rejected": -168.77499389648438, | |
| "loss": 0.1783, | |
| "rewards/accuracies": 0.7679687738418579, | |
| "rewards/chosen": 0.032080840319395065, | |
| "rewards/margins": 6.479296684265137, | |
| "rewards/rejected": -6.448828220367432, | |
| "step": 580 | |
| }, | |
| { | |
| "epoch": 1.6573033707865168, | |
| "grad_norm": 0.3948213756084442, | |
| "learning_rate": 4.3506166868781755e-08, | |
| "logits/chosen": -3.2671875953674316, | |
| "logits/rejected": -2.521484375, | |
| "logps/chosen": -41.20624923706055, | |
| "logps/rejected": -172.6875, | |
| "loss": 0.1759, | |
| "rewards/accuracies": 0.76953125, | |
| "rewards/chosen": 0.012204742059111595, | |
| "rewards/margins": 6.639843940734863, | |
| "rewards/rejected": -6.629296779632568, | |
| "step": 590 | |
| }, | |
| { | |
| "epoch": 1.6853932584269664, | |
| "grad_norm": 0.4554503560066223, | |
| "learning_rate": 3.683995891147695e-08, | |
| "logits/chosen": -3.246289014816284, | |
| "logits/rejected": NaN, | |
| "logps/chosen": -42.18281173706055, | |
| "logps/rejected": -173.86874389648438, | |
| "loss": 0.1625, | |
| "rewards/accuracies": 0.784375011920929, | |
| "rewards/chosen": 0.0017807006370276213, | |
| "rewards/margins": 6.689453125, | |
| "rewards/rejected": -6.688672065734863, | |
| "step": 600 | |
| }, | |
| { | |
| "epoch": 1.7134831460674156, | |
| "grad_norm": 0.6775197386741638, | |
| "learning_rate": 3.0687272163768986e-08, | |
| "logits/chosen": -3.259960889816284, | |
| "logits/rejected": NaN, | |
| "logps/chosen": -41.376564025878906, | |
| "logps/rejected": -174.4499969482422, | |
| "loss": 0.1619, | |
| "rewards/accuracies": 0.788281261920929, | |
| "rewards/chosen": 0.07793807983398438, | |
| "rewards/margins": 6.741796970367432, | |
| "rewards/rejected": -6.6640625, | |
| "step": 610 | |
| }, | |
| { | |
| "epoch": 1.7415730337078652, | |
| "grad_norm": 0.11740182340145111, | |
| "learning_rate": 2.5062928986944676e-08, | |
| "logits/chosen": -3.2728514671325684, | |
| "logits/rejected": NaN, | |
| "logps/chosen": -40.89374923706055, | |
| "logps/rejected": -175.0625, | |
| "loss": 0.1611, | |
| "rewards/accuracies": 0.7835937738418579, | |
| "rewards/chosen": 0.07378844916820526, | |
| "rewards/margins": 6.797656059265137, | |
| "rewards/rejected": -6.723828315734863, | |
| "step": 620 | |
| }, | |
| { | |
| "epoch": 1.7696629213483146, | |
| "grad_norm": 0.23929810523986816, | |
| "learning_rate": 1.9980478916351296e-08, | |
| "logits/chosen": -3.2544922828674316, | |
| "logits/rejected": -2.5478515625, | |
| "logps/chosen": -42.296875, | |
| "logps/rejected": -174.1374969482422, | |
| "loss": 0.1689, | |
| "rewards/accuracies": 0.785937488079071, | |
| "rewards/chosen": -0.01357345562428236, | |
| "rewards/margins": 6.648046970367432, | |
| "rewards/rejected": -6.661328315734863, | |
| "step": 630 | |
| }, | |
| { | |
| "epoch": 1.797752808988764, | |
| "grad_norm": 0.06252361834049225, | |
| "learning_rate": 1.5452166019378987e-08, | |
| "logits/chosen": -3.2582030296325684, | |
| "logits/rejected": -2.5250000953674316, | |
| "logps/chosen": -39.837501525878906, | |
| "logps/rejected": -173.9875030517578, | |
| "loss": 0.1693, | |
| "rewards/accuracies": 0.776562511920929, | |
| "rewards/chosen": 0.06287650763988495, | |
| "rewards/margins": 6.756249904632568, | |
| "rewards/rejected": -6.696875095367432, | |
| "step": 640 | |
| }, | |
| { | |
| "epoch": 1.8258426966292136, | |
| "grad_norm": 0.6466670632362366, | |
| "learning_rate": 1.1488899398429896e-08, | |
| "logits/chosen": -3.2587890625, | |
| "logits/rejected": -2.5443358421325684, | |
| "logps/chosen": -41.12812423706055, | |
| "logps/rejected": -175.3125, | |
| "loss": 0.1643, | |
| "rewards/accuracies": 0.793749988079071, | |
| "rewards/chosen": 0.057323455810546875, | |
| "rewards/margins": 6.766015529632568, | |
| "rewards/rejected": -6.710156440734863, | |
| "step": 650 | |
| }, | |
| { | |
| "epoch": 1.8539325842696628, | |
| "grad_norm": 0.364284485578537, | |
| "learning_rate": 8.100226909935059e-09, | |
| "logits/chosen": -3.2474608421325684, | |
| "logits/rejected": -2.5396485328674316, | |
| "logps/chosen": -43.040626525878906, | |
| "logps/rejected": -175.02499389648438, | |
| "loss": 0.1656, | |
| "rewards/accuracies": 0.7906249761581421, | |
| "rewards/chosen": -0.026231002062559128, | |
| "rewards/margins": 6.670312404632568, | |
| "rewards/rejected": -6.696093559265137, | |
| "step": 660 | |
| }, | |
| { | |
| "epoch": 1.8820224719101124, | |
| "grad_norm": 0.1942785233259201, | |
| "learning_rate": 5.2943121627319346e-09, | |
| "logits/chosen": -3.252734422683716, | |
| "logits/rejected": -2.544140577316284, | |
| "logps/chosen": -41.951560974121094, | |
| "logps/rejected": -170.24374389648438, | |
| "loss": 0.1726, | |
| "rewards/accuracies": 0.770312488079071, | |
| "rewards/chosen": -0.00887908972799778, | |
| "rewards/margins": 6.493359565734863, | |
| "rewards/rejected": -6.500390529632568, | |
| "step": 670 | |
| }, | |
| { | |
| "epoch": 1.9101123595505618, | |
| "grad_norm": 0.38263440132141113, | |
| "learning_rate": 3.077914851215585e-09, | |
| "logits/chosen": -3.257031202316284, | |
| "logits/rejected": -2.5390625, | |
| "logps/chosen": -41.865623474121094, | |
| "logps/rejected": -176.53750610351562, | |
| "loss": 0.1582, | |
| "rewards/accuracies": 0.796093761920929, | |
| "rewards/chosen": 0.024802017956972122, | |
| "rewards/margins": 6.796093940734863, | |
| "rewards/rejected": -6.771484375, | |
| "step": 680 | |
| }, | |
| { | |
| "epoch": 1.9382022471910112, | |
| "grad_norm": 0.4378679096698761, | |
| "learning_rate": 1.4563744706429514e-09, | |
| "logits/chosen": -3.2578125, | |
| "logits/rejected": -2.5419921875, | |
| "logps/chosen": -40.009376525878906, | |
| "logps/rejected": -171.91250610351562, | |
| "loss": 0.1703, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": 0.07464599609375, | |
| "rewards/margins": 6.642968654632568, | |
| "rewards/rejected": -6.567968845367432, | |
| "step": 690 | |
| }, | |
| { | |
| "epoch": 1.9662921348314608, | |
| "grad_norm": 0.19188769161701202, | |
| "learning_rate": 4.3359745382104405e-10, | |
| "logits/chosen": -3.271679639816284, | |
| "logits/rejected": -2.5396485328674316, | |
| "logps/chosen": -41.52031326293945, | |
| "logps/rejected": -175.375, | |
| "loss": 0.1601, | |
| "rewards/accuracies": 0.7906249761581421, | |
| "rewards/chosen": 0.01084976177662611, | |
| "rewards/margins": 6.757421970367432, | |
| "rewards/rejected": -6.750390529632568, | |
| "step": 700 | |
| }, | |
| { | |
| "epoch": 1.99438202247191, | |
| "grad_norm": 0.2792583107948303, | |
| "learning_rate": 1.2047760167999133e-11, | |
| "logits/chosen": -3.2544922828674316, | |
| "logits/rejected": -2.5517578125, | |
| "logps/chosen": -42.29999923706055, | |
| "logps/rejected": -175.3625030517578, | |
| "loss": 0.1594, | |
| "rewards/accuracies": 0.7867187261581421, | |
| "rewards/chosen": -0.014897918328642845, | |
| "rewards/margins": 6.701171875, | |
| "rewards/rejected": -6.717187404632568, | |
| "step": 710 | |
| }, | |
| { | |
| "epoch": 2.0, | |
| "step": 712, | |
| "total_flos": 0.0, | |
| "train_loss": 0.22515048312672067, | |
| "train_runtime": 2958.2669, | |
| "train_samples_per_second": 30.802, | |
| "train_steps_per_second": 0.241 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 712, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 2, | |
| "save_steps": 500, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": false, | |
| "should_training_stop": false | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 2, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |