| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.5002163565556036, | |
| "eval_steps": 500, | |
| "global_step": 289, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.0017308524448290783, | |
| "grad_norm": 46.5, | |
| "kl": 0.0, | |
| "learning_rate": 1.4285714285714287e-07, | |
| "logits/chosen": -6239313.454545454, | |
| "logits/rejected": -4940240.761904762, | |
| "logps/chosen": -236.17436079545453, | |
| "logps/rejected": -209.70107886904762, | |
| "loss": 0.5, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "epoch": 0.0034617048896581565, | |
| "grad_norm": 38.25, | |
| "kl": 0.0, | |
| "learning_rate": 2.8571428571428575e-07, | |
| "logits/chosen": -2665428.3076923075, | |
| "logits/rejected": -1073632.5263157894, | |
| "logps/chosen": -155.0839562049279, | |
| "logps/rejected": -255.23524876644737, | |
| "loss": 0.5, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 2 | |
| }, | |
| { | |
| "epoch": 0.005192557334487235, | |
| "grad_norm": 39.25, | |
| "kl": 0.11506986618041992, | |
| "learning_rate": 4.285714285714286e-07, | |
| "logits/chosen": -1627763.2, | |
| "logits/rejected": -1337906.8235294118, | |
| "logps/chosen": -214.50914713541667, | |
| "logps/rejected": -210.7369887408088, | |
| "loss": 0.5011, | |
| "rewards/chosen": -0.005750782291094462, | |
| "rewards/margins": -0.006572681913773219, | |
| "rewards/rejected": 0.0008218996226787567, | |
| "step": 3 | |
| }, | |
| { | |
| "epoch": 0.006923409779316313, | |
| "grad_norm": 33.5, | |
| "kl": 0.13516950607299805, | |
| "learning_rate": 5.714285714285715e-07, | |
| "logits/chosen": -9900144.0, | |
| "logits/rejected": 2390790.4, | |
| "logps/chosen": -236.11850873161765, | |
| "logps/rejected": -187.40989583333334, | |
| "loss": 0.5052, | |
| "rewards/chosen": 0.00019769019940320184, | |
| "rewards/margins": -0.03652356716932035, | |
| "rewards/rejected": 0.03672125736872355, | |
| "step": 4 | |
| }, | |
| { | |
| "epoch": 0.00865426222414539, | |
| "grad_norm": 39.0, | |
| "kl": 0.16583895683288574, | |
| "learning_rate": 7.142857142857143e-07, | |
| "logits/chosen": 273089.5, | |
| "logits/rejected": -8537414.0, | |
| "logps/chosen": -191.99412536621094, | |
| "logps/rejected": -250.5272216796875, | |
| "loss": 0.5, | |
| "rewards/chosen": -0.018333029001951218, | |
| "rewards/margins": -0.014739224454388022, | |
| "rewards/rejected": -0.0035938045475631952, | |
| "step": 5 | |
| }, | |
| { | |
| "epoch": 0.01038511466897447, | |
| "grad_norm": 37.0, | |
| "kl": 0.1269383430480957, | |
| "learning_rate": 8.571428571428572e-07, | |
| "logits/chosen": 625734.125, | |
| "logits/rejected": -3864760.5, | |
| "logps/chosen": -130.1186065673828, | |
| "logps/rejected": -263.1868591308594, | |
| "loss": 0.4974, | |
| "rewards/chosen": 0.005134785547852516, | |
| "rewards/margins": 0.02763364464044571, | |
| "rewards/rejected": -0.022498859092593193, | |
| "step": 6 | |
| }, | |
| { | |
| "epoch": 0.012115967113803548, | |
| "grad_norm": 45.5, | |
| "kl": 0.058301448822021484, | |
| "learning_rate": 1.0000000000000002e-06, | |
| "logits/chosen": 11427891.2, | |
| "logits/rejected": -8692000.94117647, | |
| "logps/chosen": -247.42198893229167, | |
| "logps/rejected": -318.3026769301471, | |
| "loss": 0.4879, | |
| "rewards/chosen": 0.020061949888865154, | |
| "rewards/margins": 0.09507267031015135, | |
| "rewards/rejected": -0.07501072042128619, | |
| "step": 7 | |
| }, | |
| { | |
| "epoch": 0.013846819558632626, | |
| "grad_norm": 34.75, | |
| "kl": 0.10779595375061035, | |
| "learning_rate": 1.142857142857143e-06, | |
| "logits/chosen": 9745310.315789474, | |
| "logits/rejected": 4968272.0, | |
| "logps/chosen": -266.39432565789474, | |
| "logps/rejected": -200.4144568810096, | |
| "loss": 0.4926, | |
| "rewards/chosen": 0.03307872383218063, | |
| "rewards/margins": 0.08410431619597833, | |
| "rewards/rejected": -0.0510255923637977, | |
| "step": 8 | |
| }, | |
| { | |
| "epoch": 0.015577672003461706, | |
| "grad_norm": 35.75, | |
| "kl": 0.0004693269729614258, | |
| "learning_rate": 1.2857142857142856e-06, | |
| "logits/chosen": 2292229.3333333335, | |
| "logits/rejected": -5866576.571428572, | |
| "logps/chosen": -164.06934950086804, | |
| "logps/rejected": -255.37636021205358, | |
| "loss": 0.4857, | |
| "rewards/chosen": 0.017767790291044448, | |
| "rewards/margins": 0.12696768035964362, | |
| "rewards/rejected": -0.10919989006859916, | |
| "step": 9 | |
| }, | |
| { | |
| "epoch": 0.01730852444829078, | |
| "grad_norm": 58.0, | |
| "kl": 0.08134031295776367, | |
| "learning_rate": 1.4285714285714286e-06, | |
| "logits/chosen": -2206551.1428571427, | |
| "logits/rejected": -2327785.3333333335, | |
| "logps/chosen": -210.65478515625, | |
| "logps/rejected": -399.8186848958333, | |
| "loss": 0.4595, | |
| "rewards/chosen": 0.041608184576034546, | |
| "rewards/margins": 0.30469969577259487, | |
| "rewards/rejected": -0.2630915111965603, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.019039376893119863, | |
| "grad_norm": 33.0, | |
| "kl": 0.06869983673095703, | |
| "learning_rate": 1.5714285714285714e-06, | |
| "logits/chosen": 5004582.315789473, | |
| "logits/rejected": 15390077.538461538, | |
| "logps/chosen": -168.22392835115133, | |
| "logps/rejected": -250.3277869591346, | |
| "loss": 0.4842, | |
| "rewards/chosen": 0.025214639149214093, | |
| "rewards/margins": 0.15980646617499442, | |
| "rewards/rejected": -0.13459182702578032, | |
| "step": 11 | |
| }, | |
| { | |
| "epoch": 0.02077022933794894, | |
| "grad_norm": 35.25, | |
| "kl": 0.03198128938674927, | |
| "learning_rate": 1.7142857142857145e-06, | |
| "logits/chosen": -6019438.5, | |
| "logits/rejected": -12351150.0, | |
| "logps/chosen": -203.1639404296875, | |
| "logps/rejected": -248.1376495361328, | |
| "loss": 0.4481, | |
| "rewards/chosen": 0.022723043337464333, | |
| "rewards/margins": 0.46131726540625095, | |
| "rewards/rejected": -0.4385942220687866, | |
| "step": 12 | |
| }, | |
| { | |
| "epoch": 0.02250108178277802, | |
| "grad_norm": 27.375, | |
| "kl": 0.03030562400817871, | |
| "learning_rate": 1.8571428571428573e-06, | |
| "logits/chosen": -1771344.705882353, | |
| "logits/rejected": 4848613.333333333, | |
| "logps/chosen": -156.79733455882354, | |
| "logps/rejected": -143.33746744791668, | |
| "loss": 0.4641, | |
| "rewards/chosen": 0.040735574329600614, | |
| "rewards/margins": 0.31227434055477965, | |
| "rewards/rejected": -0.27153876622517903, | |
| "step": 13 | |
| }, | |
| { | |
| "epoch": 0.024231934227607096, | |
| "grad_norm": 31.75, | |
| "kl": 0.0, | |
| "learning_rate": 2.0000000000000003e-06, | |
| "logits/chosen": -402277.15789473685, | |
| "logits/rejected": 8271367.384615385, | |
| "logps/chosen": -283.14478824013156, | |
| "logps/rejected": -204.32831280048077, | |
| "loss": 0.4584, | |
| "rewards/chosen": 0.008756919126761587, | |
| "rewards/margins": 0.42188657892619064, | |
| "rewards/rejected": -0.41312965979942906, | |
| "step": 14 | |
| }, | |
| { | |
| "epoch": 0.025962786672436174, | |
| "grad_norm": 36.75, | |
| "kl": 0.0032711029052734375, | |
| "learning_rate": 2.1428571428571427e-06, | |
| "logits/chosen": -811063.5882352941, | |
| "logits/rejected": -12282100.266666668, | |
| "logps/chosen": -176.79848345588235, | |
| "logps/rejected": -325.2669270833333, | |
| "loss": 0.3948, | |
| "rewards/chosen": 0.08565068244934082, | |
| "rewards/margins": 0.9919464588165283, | |
| "rewards/rejected": -0.9062957763671875, | |
| "step": 15 | |
| }, | |
| { | |
| "epoch": 0.027693639117265252, | |
| "grad_norm": 30.375, | |
| "kl": 0.007568359375, | |
| "learning_rate": 2.285714285714286e-06, | |
| "logits/chosen": -1084260.0, | |
| "logits/rejected": 31534450.666666668, | |
| "logps/chosen": -210.231787109375, | |
| "logps/rejected": -276.17873128255206, | |
| "loss": 0.4251, | |
| "rewards/chosen": 0.08407727479934693, | |
| "rewards/margins": 0.8169409394264221, | |
| "rewards/rejected": -0.7328636646270752, | |
| "step": 16 | |
| }, | |
| { | |
| "epoch": 0.02942449156209433, | |
| "grad_norm": 30.0, | |
| "kl": 0.0, | |
| "learning_rate": 2.428571428571429e-06, | |
| "logits/chosen": -1036163.6923076923, | |
| "logits/rejected": 961232.6315789474, | |
| "logps/chosen": -268.2616624098558, | |
| "logps/rejected": -225.40373149671052, | |
| "loss": 0.4054, | |
| "rewards/chosen": 0.010549396276473999, | |
| "rewards/margins": 0.7161211230252919, | |
| "rewards/rejected": -0.7055717267488179, | |
| "step": 17 | |
| }, | |
| { | |
| "epoch": 0.03115534400692341, | |
| "grad_norm": 26.25, | |
| "kl": 0.0, | |
| "learning_rate": 2.571428571428571e-06, | |
| "logits/chosen": 4818987.555555556, | |
| "logits/rejected": -827468.8571428572, | |
| "logps/chosen": -251.50027126736111, | |
| "logps/rejected": -213.26834542410714, | |
| "loss": 0.4032, | |
| "rewards/chosen": 0.03222567505306668, | |
| "rewards/margins": 1.113093238028269, | |
| "rewards/rejected": -1.0808675629752023, | |
| "step": 18 | |
| }, | |
| { | |
| "epoch": 0.03288619645175249, | |
| "grad_norm": 24.625, | |
| "kl": 0.0, | |
| "learning_rate": 2.7142857142857144e-06, | |
| "logits/chosen": -1698852.380952381, | |
| "logits/rejected": 2662217.6363636362, | |
| "logps/chosen": -172.64027622767858, | |
| "logps/rejected": -205.12626509232953, | |
| "loss": 0.4404, | |
| "rewards/chosen": -0.05192979744502476, | |
| "rewards/margins": 0.9354158980505807, | |
| "rewards/rejected": -0.9873456954956055, | |
| "step": 19 | |
| }, | |
| { | |
| "epoch": 0.03461704889658156, | |
| "grad_norm": 22.375, | |
| "kl": 0.0, | |
| "learning_rate": 2.8571428571428573e-06, | |
| "logits/chosen": 6553660.0, | |
| "logits/rejected": -5781368.444444444, | |
| "logps/chosen": -143.31090436662947, | |
| "logps/rejected": -192.53050401475696, | |
| "loss": 0.3749, | |
| "rewards/chosen": 0.09408829041889735, | |
| "rewards/margins": 1.171789647094787, | |
| "rewards/rejected": -1.0777013566758897, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.036347901341410645, | |
| "grad_norm": 20.375, | |
| "kl": 0.012153387069702148, | |
| "learning_rate": 3e-06, | |
| "logits/chosen": 1939333.8666666667, | |
| "logits/rejected": 1052395.0588235294, | |
| "logps/chosen": -177.10651041666668, | |
| "logps/rejected": -190.28341854319854, | |
| "loss": 0.3776, | |
| "rewards/chosen": 0.045921965440114336, | |
| "rewards/margins": 1.3893623017797283, | |
| "rewards/rejected": -1.343440336339614, | |
| "step": 21 | |
| }, | |
| { | |
| "epoch": 0.038078753786239726, | |
| "grad_norm": 21.0, | |
| "kl": 0.0, | |
| "learning_rate": 3.142857142857143e-06, | |
| "logits/chosen": 3900064.5, | |
| "logits/rejected": 2436417.0, | |
| "logps/chosen": -188.06832885742188, | |
| "logps/rejected": -307.90692138671875, | |
| "loss": 0.3542, | |
| "rewards/chosen": -0.08758784085512161, | |
| "rewards/margins": 1.7587207481265068, | |
| "rewards/rejected": -1.8463085889816284, | |
| "step": 22 | |
| }, | |
| { | |
| "epoch": 0.0398096062310688, | |
| "grad_norm": 22.625, | |
| "kl": 0.000914454460144043, | |
| "learning_rate": 3.285714285714286e-06, | |
| "logits/chosen": -42746.86666666667, | |
| "logits/rejected": 1372338.8235294118, | |
| "logps/chosen": -226.461865234375, | |
| "logps/rejected": -258.68396714154414, | |
| "loss": 0.3754, | |
| "rewards/chosen": -0.14824188550313314, | |
| "rewards/margins": 1.6312208166309432, | |
| "rewards/rejected": -1.7794627021340763, | |
| "step": 23 | |
| }, | |
| { | |
| "epoch": 0.04154045867589788, | |
| "grad_norm": 19.75, | |
| "kl": 0.0, | |
| "learning_rate": 3.428571428571429e-06, | |
| "logits/chosen": -1758418.3333333333, | |
| "logits/rejected": 7633656.0, | |
| "logps/chosen": -143.17743598090277, | |
| "logps/rejected": -136.41956438337053, | |
| "loss": 0.4083, | |
| "rewards/chosen": -0.15943604045444065, | |
| "rewards/margins": 1.379212019935487, | |
| "rewards/rejected": -1.5386480603899275, | |
| "step": 24 | |
| }, | |
| { | |
| "epoch": 0.043271311120726956, | |
| "grad_norm": 22.375, | |
| "kl": 0.0, | |
| "learning_rate": 3.5714285714285718e-06, | |
| "logits/chosen": -3627161.777777778, | |
| "logits/rejected": -1282915.142857143, | |
| "logps/chosen": -189.07590060763889, | |
| "logps/rejected": -282.23025948660717, | |
| "loss": 0.3667, | |
| "rewards/chosen": -0.05861267778608534, | |
| "rewards/margins": 2.27708803850507, | |
| "rewards/rejected": -2.335700716291155, | |
| "step": 25 | |
| }, | |
| { | |
| "epoch": 0.04500216356555604, | |
| "grad_norm": 20.125, | |
| "kl": 0.0, | |
| "learning_rate": 3.7142857142857146e-06, | |
| "logits/chosen": 3197816.8571428573, | |
| "logits/rejected": -24990.666666666668, | |
| "logps/chosen": -133.38133893694197, | |
| "logps/rejected": -252.71739366319446, | |
| "loss": 0.3729, | |
| "rewards/chosen": -0.2894209793635777, | |
| "rewards/margins": 1.5688878127506802, | |
| "rewards/rejected": -1.8583087921142578, | |
| "step": 26 | |
| }, | |
| { | |
| "epoch": 0.04673301601038511, | |
| "grad_norm": 22.0, | |
| "kl": 0.0, | |
| "learning_rate": 3.857142857142858e-06, | |
| "logits/chosen": 8530238.857142856, | |
| "logits/rejected": -6356856.888888889, | |
| "logps/chosen": -228.37095424107142, | |
| "logps/rejected": -290.1540256076389, | |
| "loss": 0.3187, | |
| "rewards/chosen": -0.04233703442982265, | |
| "rewards/margins": 2.3515855594286843, | |
| "rewards/rejected": -2.393922593858507, | |
| "step": 27 | |
| }, | |
| { | |
| "epoch": 0.04846386845521419, | |
| "grad_norm": 17.125, | |
| "kl": 0.0, | |
| "learning_rate": 4.000000000000001e-06, | |
| "logits/chosen": 11541931.294117646, | |
| "logits/rejected": -18899741.866666667, | |
| "logps/chosen": -161.64244887408088, | |
| "logps/rejected": -298.1225911458333, | |
| "loss": 0.3221, | |
| "rewards/chosen": -0.20178667236776912, | |
| "rewards/margins": 3.408641692703845, | |
| "rewards/rejected": -3.6104283650716145, | |
| "step": 28 | |
| }, | |
| { | |
| "epoch": 0.050194720900043274, | |
| "grad_norm": 21.125, | |
| "kl": 0.0, | |
| "learning_rate": 4.1428571428571435e-06, | |
| "logits/chosen": -8176106.0, | |
| "logits/rejected": 8892046.0, | |
| "logps/chosen": -202.91030883789062, | |
| "logps/rejected": -320.7770690917969, | |
| "loss": 0.3603, | |
| "rewards/chosen": -0.17716556787490845, | |
| "rewards/margins": 2.8113109469413757, | |
| "rewards/rejected": -2.988476514816284, | |
| "step": 29 | |
| }, | |
| { | |
| "epoch": 0.05192557334487235, | |
| "grad_norm": 20.75, | |
| "kl": 0.0, | |
| "learning_rate": 4.2857142857142855e-06, | |
| "logits/chosen": 1024021.4736842106, | |
| "logits/rejected": 4154760.0, | |
| "logps/chosen": -166.43669048108552, | |
| "logps/rejected": -148.49478853665866, | |
| "loss": 0.4417, | |
| "rewards/chosen": -0.38098611329731186, | |
| "rewards/margins": 1.4392063704579465, | |
| "rewards/rejected": -1.8201924837552583, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.05365642578970143, | |
| "grad_norm": 19.375, | |
| "kl": 0.0, | |
| "learning_rate": 4.428571428571429e-06, | |
| "logits/chosen": 12327768.727272727, | |
| "logits/rejected": -4335401.904761905, | |
| "logps/chosen": -141.8250732421875, | |
| "logps/rejected": -224.83528645833334, | |
| "loss": 0.3034, | |
| "rewards/chosen": -0.04076832803812894, | |
| "rewards/margins": 2.3152393841898284, | |
| "rewards/rejected": -2.3560077122279575, | |
| "step": 31 | |
| }, | |
| { | |
| "epoch": 0.055387278234530504, | |
| "grad_norm": 17.125, | |
| "kl": 0.0, | |
| "learning_rate": 4.571428571428572e-06, | |
| "logits/chosen": -7621606.545454546, | |
| "logits/rejected": -5030670.857142857, | |
| "logps/chosen": -168.98353160511363, | |
| "logps/rejected": -240.65597098214286, | |
| "loss": 0.294, | |
| "rewards/chosen": -0.1550229029221968, | |
| "rewards/margins": 2.5555503843150613, | |
| "rewards/rejected": -2.710573287237258, | |
| "step": 32 | |
| }, | |
| { | |
| "epoch": 0.057118130679359586, | |
| "grad_norm": 20.75, | |
| "kl": 0.0, | |
| "learning_rate": 4.714285714285715e-06, | |
| "logits/chosen": -4837878.153846154, | |
| "logits/rejected": -2727320.4210526315, | |
| "logps/chosen": -260.5615985576923, | |
| "logps/rejected": -309.63633326480266, | |
| "loss": 0.3038, | |
| "rewards/chosen": -0.1790018998659574, | |
| "rewards/margins": 3.2555804590464605, | |
| "rewards/rejected": -3.434582358912418, | |
| "step": 33 | |
| }, | |
| { | |
| "epoch": 0.05884898312418866, | |
| "grad_norm": 16.75, | |
| "kl": 0.0, | |
| "learning_rate": 4.857142857142858e-06, | |
| "logits/chosen": 334761.73333333334, | |
| "logits/rejected": -6532100.705882353, | |
| "logps/chosen": -111.3039794921875, | |
| "logps/rejected": -291.9061638327206, | |
| "loss": 0.3181, | |
| "rewards/chosen": -0.061692579587300615, | |
| "rewards/margins": 3.2174934447980394, | |
| "rewards/rejected": -3.27918602438534, | |
| "step": 34 | |
| }, | |
| { | |
| "epoch": 0.06057983556901774, | |
| "grad_norm": 19.375, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -5367810.0, | |
| "logits/rejected": -11132618.0, | |
| "logps/chosen": -157.48095703125, | |
| "logps/rejected": -343.7468566894531, | |
| "loss": 0.3208, | |
| "rewards/chosen": -0.24039308726787567, | |
| "rewards/margins": 3.439171150326729, | |
| "rewards/rejected": -3.6795642375946045, | |
| "step": 35 | |
| }, | |
| { | |
| "epoch": 0.06231068801384682, | |
| "grad_norm": 21.625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -4363456.888888889, | |
| "logits/rejected": -3689830.285714286, | |
| "logps/chosen": -200.380126953125, | |
| "logps/rejected": -326.18729073660717, | |
| "loss": 0.3453, | |
| "rewards/chosen": -0.15904908710055882, | |
| "rewards/margins": 3.3486854840838722, | |
| "rewards/rejected": -3.507734571184431, | |
| "step": 36 | |
| }, | |
| { | |
| "epoch": 0.0640415404586759, | |
| "grad_norm": 24.625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -14376078.4, | |
| "logits/rejected": -13662346.666666666, | |
| "logps/chosen": -232.297265625, | |
| "logps/rejected": -266.85874430338544, | |
| "loss": 0.3776, | |
| "rewards/chosen": -0.15787798166275024, | |
| "rewards/margins": 3.1688521107037864, | |
| "rewards/rejected": -3.3267300923665366, | |
| "step": 37 | |
| }, | |
| { | |
| "epoch": 0.06577239290350498, | |
| "grad_norm": 19.375, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 9234006.4, | |
| "logits/rejected": 8716803.764705881, | |
| "logps/chosen": -195.22076822916668, | |
| "logps/rejected": -223.59991096047793, | |
| "loss": 0.3266, | |
| "rewards/chosen": 0.033547862370808916, | |
| "rewards/margins": 2.3299624059714525, | |
| "rewards/rejected": -2.2964145436006436, | |
| "step": 38 | |
| }, | |
| { | |
| "epoch": 0.06750324534833406, | |
| "grad_norm": 23.5, | |
| "kl": 0.22034478187561035, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -10664284.0, | |
| "logits/rejected": -6822504.5, | |
| "logps/chosen": -254.58663940429688, | |
| "logps/rejected": -303.74578857421875, | |
| "loss": 0.3069, | |
| "rewards/chosen": -0.025931095704436302, | |
| "rewards/margins": 2.927818799391389, | |
| "rewards/rejected": -2.953749895095825, | |
| "step": 39 | |
| }, | |
| { | |
| "epoch": 0.06923409779316313, | |
| "grad_norm": 19.75, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -3069599.6666666665, | |
| "logits/rejected": -6284954.8, | |
| "logps/chosen": -277.9341634114583, | |
| "logps/rejected": -314.748974609375, | |
| "loss": 0.2888, | |
| "rewards/chosen": -0.13368964195251465, | |
| "rewards/margins": 2.755869913101196, | |
| "rewards/rejected": -2.8895595550537108, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.07096495023799221, | |
| "grad_norm": 22.0, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 946472.0, | |
| "logits/rejected": 9102588.235294119, | |
| "logps/chosen": -248.21149088541668, | |
| "logps/rejected": -162.85635914522058, | |
| "loss": 0.3098, | |
| "rewards/chosen": 0.07899113496144612, | |
| "rewards/margins": 2.07641756067089, | |
| "rewards/rejected": -1.9974264257094438, | |
| "step": 41 | |
| }, | |
| { | |
| "epoch": 0.07269580268282129, | |
| "grad_norm": 21.0, | |
| "kl": 0.07075059413909912, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 7388380.19047619, | |
| "logits/rejected": -89609.45454545454, | |
| "logps/chosen": -191.81854538690476, | |
| "logps/rejected": -390.0617009943182, | |
| "loss": 0.3575, | |
| "rewards/chosen": 0.03016080175127302, | |
| "rewards/margins": 3.0582882986440287, | |
| "rewards/rejected": -3.028127496892756, | |
| "step": 42 | |
| }, | |
| { | |
| "epoch": 0.07442665512765037, | |
| "grad_norm": 22.125, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -6876745.6, | |
| "logits/rejected": -1067452.5, | |
| "logps/chosen": -200.07301025390626, | |
| "logps/rejected": -290.6153564453125, | |
| "loss": 0.3613, | |
| "rewards/chosen": -0.03498818874359131, | |
| "rewards/margins": 2.9018725315729776, | |
| "rewards/rejected": -2.936860720316569, | |
| "step": 43 | |
| }, | |
| { | |
| "epoch": 0.07615750757247945, | |
| "grad_norm": 20.5, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 3785227.111111111, | |
| "logits/rejected": 1574901.0, | |
| "logps/chosen": -188.12406412760416, | |
| "logps/rejected": -150.00552804129464, | |
| "loss": 0.3361, | |
| "rewards/chosen": 0.11941173341539171, | |
| "rewards/margins": 2.0707845612177773, | |
| "rewards/rejected": -1.9513728278023856, | |
| "step": 44 | |
| }, | |
| { | |
| "epoch": 0.07788836001730852, | |
| "grad_norm": 18.125, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -3288104.0, | |
| "logits/rejected": -1550288.125, | |
| "logps/chosen": -169.7630157470703, | |
| "logps/rejected": -244.53619384765625, | |
| "loss": 0.3335, | |
| "rewards/chosen": -0.1161470040678978, | |
| "rewards/margins": 2.0714645758271217, | |
| "rewards/rejected": -2.1876115798950195, | |
| "step": 45 | |
| }, | |
| { | |
| "epoch": 0.0796192124621376, | |
| "grad_norm": 22.375, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 2523540.222222222, | |
| "logits/rejected": 807282.2142857143, | |
| "logps/chosen": -206.785400390625, | |
| "logps/rejected": -197.57388741629464, | |
| "loss": 0.3727, | |
| "rewards/chosen": -0.07473884688483344, | |
| "rewards/margins": 1.7757124862973652, | |
| "rewards/rejected": -1.8504513331821986, | |
| "step": 46 | |
| }, | |
| { | |
| "epoch": 0.08135006490696668, | |
| "grad_norm": 22.875, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -3043348.533333333, | |
| "logits/rejected": 10325686.588235294, | |
| "logps/chosen": -231.226220703125, | |
| "logps/rejected": -169.99207261029412, | |
| "loss": 0.3148, | |
| "rewards/chosen": 0.14197413126627603, | |
| "rewards/margins": 2.014080571193321, | |
| "rewards/rejected": -1.872106439927045, | |
| "step": 47 | |
| }, | |
| { | |
| "epoch": 0.08308091735179576, | |
| "grad_norm": 18.375, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 1849486.9411764706, | |
| "logits/rejected": -2160865.3333333335, | |
| "logps/chosen": -169.53436638327207, | |
| "logps/rejected": -239.082568359375, | |
| "loss": 0.3451, | |
| "rewards/chosen": -0.1786177158355713, | |
| "rewards/margins": 2.253294515609741, | |
| "rewards/rejected": -2.4319122314453123, | |
| "step": 48 | |
| }, | |
| { | |
| "epoch": 0.08481176979662484, | |
| "grad_norm": 21.125, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 900298.5714285715, | |
| "logits/rejected": -6812352.7272727275, | |
| "logps/chosen": -166.2104259672619, | |
| "logps/rejected": -353.5106312144886, | |
| "loss": 0.3726, | |
| "rewards/chosen": -0.07294606594812303, | |
| "rewards/margins": 2.6174716572740895, | |
| "rewards/rejected": -2.6904177232222124, | |
| "step": 49 | |
| }, | |
| { | |
| "epoch": 0.08654262224145391, | |
| "grad_norm": 19.125, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -9324354.461538462, | |
| "logits/rejected": -14040514.52631579, | |
| "logps/chosen": -223.0580115685096, | |
| "logps/rejected": -286.9508634868421, | |
| "loss": 0.2674, | |
| "rewards/chosen": 0.13786140772012564, | |
| "rewards/margins": 2.67845962452985, | |
| "rewards/rejected": -2.5405982168097245, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.088273474686283, | |
| "grad_norm": 20.125, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 930246.5, | |
| "logits/rejected": -7422215.0, | |
| "logps/chosen": -238.2467803955078, | |
| "logps/rejected": -259.29217529296875, | |
| "loss": 0.294, | |
| "rewards/chosen": 0.032952681183815, | |
| "rewards/margins": 3.1361082941293716, | |
| "rewards/rejected": -3.1031556129455566, | |
| "step": 51 | |
| }, | |
| { | |
| "epoch": 0.09000432713111207, | |
| "grad_norm": 20.625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 2028204.625, | |
| "logits/rejected": -1362418.25, | |
| "logps/chosen": -190.04937744140625, | |
| "logps/rejected": -216.1826171875, | |
| "loss": 0.3165, | |
| "rewards/chosen": -0.050155334174633026, | |
| "rewards/margins": 2.5483616068959236, | |
| "rewards/rejected": -2.5985169410705566, | |
| "step": 52 | |
| }, | |
| { | |
| "epoch": 0.09173517957594116, | |
| "grad_norm": 26.875, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 1491913.3333333333, | |
| "logits/rejected": 17979700.363636363, | |
| "logps/chosen": -285.51971726190476, | |
| "logps/rejected": -230.9169256036932, | |
| "loss": 0.3878, | |
| "rewards/chosen": -0.13681457156226748, | |
| "rewards/margins": 2.993390062670687, | |
| "rewards/rejected": -3.1302046342329546, | |
| "step": 53 | |
| }, | |
| { | |
| "epoch": 0.09346603202077022, | |
| "grad_norm": 16.0, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -5621796.571428572, | |
| "logits/rejected": -3721314.222222222, | |
| "logps/chosen": -154.634521484375, | |
| "logps/rejected": -144.23682996961804, | |
| "loss": 0.3166, | |
| "rewards/chosen": 0.15212011337280273, | |
| "rewards/margins": 2.1092937787373858, | |
| "rewards/rejected": -1.9571736653645833, | |
| "step": 54 | |
| }, | |
| { | |
| "epoch": 0.0951968844655993, | |
| "grad_norm": 20.25, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -13213840.0, | |
| "logits/rejected": -2418853.6, | |
| "logps/chosen": -264.71543375651044, | |
| "logps/rejected": -299.3638916015625, | |
| "loss": 0.2498, | |
| "rewards/chosen": 0.169629176457723, | |
| "rewards/margins": 2.8914440949757894, | |
| "rewards/rejected": -2.7218149185180662, | |
| "step": 55 | |
| }, | |
| { | |
| "epoch": 0.09692773691042839, | |
| "grad_norm": 20.75, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 1270494.2222222222, | |
| "logits/rejected": 3427853.1428571427, | |
| "logps/chosen": -218.35677083333334, | |
| "logps/rejected": -151.46371023995536, | |
| "loss": 0.3539, | |
| "rewards/chosen": 0.18590817186567518, | |
| "rewards/margins": 1.9470306029395452, | |
| "rewards/rejected": -1.76112243107387, | |
| "step": 56 | |
| }, | |
| { | |
| "epoch": 0.09865858935525747, | |
| "grad_norm": 19.625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 960138.5, | |
| "logits/rejected": -11440672.0, | |
| "logps/chosen": -194.89010620117188, | |
| "logps/rejected": -279.1370544433594, | |
| "loss": 0.3033, | |
| "rewards/chosen": 0.042164143174886703, | |
| "rewards/margins": 2.7914009653031826, | |
| "rewards/rejected": -2.749236822128296, | |
| "step": 57 | |
| }, | |
| { | |
| "epoch": 0.10038944180008655, | |
| "grad_norm": 17.0, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -3341974.933333333, | |
| "logits/rejected": -3835131.7647058824, | |
| "logps/chosen": -99.36764322916666, | |
| "logps/rejected": -268.3916015625, | |
| "loss": 0.3016, | |
| "rewards/chosen": -0.05830394426981608, | |
| "rewards/margins": 2.6891182179544484, | |
| "rewards/rejected": -2.7474221622242645, | |
| "step": 58 | |
| }, | |
| { | |
| "epoch": 0.10212029424491562, | |
| "grad_norm": 20.875, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -5338708.2105263155, | |
| "logits/rejected": -522996.92307692306, | |
| "logps/chosen": -177.21720805921052, | |
| "logps/rejected": -242.44989483173077, | |
| "loss": 0.3425, | |
| "rewards/chosen": -0.03957033157348633, | |
| "rewards/margins": 3.2665699812082143, | |
| "rewards/rejected": -3.3061403127817006, | |
| "step": 59 | |
| }, | |
| { | |
| "epoch": 0.1038511466897447, | |
| "grad_norm": 17.375, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -4021004.5714285714, | |
| "logits/rejected": 3649617.3333333335, | |
| "logps/chosen": -150.47998046875, | |
| "logps/rejected": -242.73616536458334, | |
| "loss": 0.2824, | |
| "rewards/chosen": 0.04531372019222805, | |
| "rewards/margins": 2.6572553756691164, | |
| "rewards/rejected": -2.611941655476888, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.10558199913457378, | |
| "grad_norm": 20.0, | |
| "kl": 0.0002346038818359375, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -4345515.0, | |
| "logits/rejected": -8544926.0, | |
| "logps/chosen": -253.4796142578125, | |
| "logps/rejected": -246.94094848632812, | |
| "loss": 0.2793, | |
| "rewards/chosen": 0.15215471386909485, | |
| "rewards/margins": 3.54859259724617, | |
| "rewards/rejected": -3.396437883377075, | |
| "step": 61 | |
| }, | |
| { | |
| "epoch": 0.10731285157940286, | |
| "grad_norm": 18.0, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -1196661.3333333333, | |
| "logits/rejected": -3318421.1428571427, | |
| "logps/chosen": -154.80669487847223, | |
| "logps/rejected": -289.3799525669643, | |
| "loss": 0.3092, | |
| "rewards/chosen": 0.0665718052122328, | |
| "rewards/margins": 3.497844584404476, | |
| "rewards/rejected": -3.4312727791922435, | |
| "step": 62 | |
| }, | |
| { | |
| "epoch": 0.10904370402423194, | |
| "grad_norm": 20.25, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -3965800.4444444445, | |
| "logits/rejected": -11330329.142857144, | |
| "logps/chosen": -182.16238064236111, | |
| "logps/rejected": -249.26039341517858, | |
| "loss": 0.3376, | |
| "rewards/chosen": -0.1392565303378635, | |
| "rewards/margins": 3.219647899506584, | |
| "rewards/rejected": -3.3589044298444475, | |
| "step": 63 | |
| }, | |
| { | |
| "epoch": 0.11077455646906101, | |
| "grad_norm": 19.875, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -2474531.25, | |
| "logits/rejected": -5974583.5, | |
| "logps/chosen": -209.27244567871094, | |
| "logps/rejected": -302.0538635253906, | |
| "loss": 0.325, | |
| "rewards/chosen": 0.020520292222499847, | |
| "rewards/margins": 3.1491325721144676, | |
| "rewards/rejected": -3.1286122798919678, | |
| "step": 64 | |
| }, | |
| { | |
| "epoch": 0.11250540891389009, | |
| "grad_norm": 14.5625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -5467615.384615385, | |
| "logits/rejected": -3365480.0, | |
| "logps/chosen": -144.33649151141827, | |
| "logps/rejected": -279.4246761924342, | |
| "loss": 0.2657, | |
| "rewards/chosen": -0.11599624156951904, | |
| "rewards/margins": 3.2443882477910897, | |
| "rewards/rejected": -3.3603844893606087, | |
| "step": 65 | |
| }, | |
| { | |
| "epoch": 0.11423626135871917, | |
| "grad_norm": 22.375, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -1735104.4, | |
| "logits/rejected": 5294354.666666667, | |
| "logps/chosen": -211.195068359375, | |
| "logps/rejected": -247.84977213541666, | |
| "loss": 0.3939, | |
| "rewards/chosen": -0.25041675567626953, | |
| "rewards/margins": 3.1013142267862954, | |
| "rewards/rejected": -3.351730982462565, | |
| "step": 66 | |
| }, | |
| { | |
| "epoch": 0.11596711380354825, | |
| "grad_norm": 19.125, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -5322431.555555556, | |
| "logits/rejected": -17504265.14285714, | |
| "logps/chosen": -156.61607530381946, | |
| "logps/rejected": -485.01771763392856, | |
| "loss": 0.3061, | |
| "rewards/chosen": 0.008888012833065458, | |
| "rewards/margins": 3.940254797065069, | |
| "rewards/rejected": -3.9313667842320035, | |
| "step": 67 | |
| }, | |
| { | |
| "epoch": 0.11769796624837732, | |
| "grad_norm": 19.125, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 2996452.2666666666, | |
| "logits/rejected": 11716244.705882354, | |
| "logps/chosen": -192.19383138020834, | |
| "logps/rejected": -302.7353515625, | |
| "loss": 0.2934, | |
| "rewards/chosen": 0.03241715629895528, | |
| "rewards/margins": 2.8701628228028615, | |
| "rewards/rejected": -2.8377456665039062, | |
| "step": 68 | |
| }, | |
| { | |
| "epoch": 0.1194288186932064, | |
| "grad_norm": 17.5, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 14927777.142857144, | |
| "logits/rejected": -4720057.777777778, | |
| "logps/chosen": -149.42860630580358, | |
| "logps/rejected": -217.70494249131946, | |
| "loss": 0.2898, | |
| "rewards/chosen": -0.16934810365949357, | |
| "rewards/margins": 3.1008769973875983, | |
| "rewards/rejected": -3.270225101047092, | |
| "step": 69 | |
| }, | |
| { | |
| "epoch": 0.12115967113803548, | |
| "grad_norm": 20.375, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 44536797.538461536, | |
| "logits/rejected": -7564164.2105263155, | |
| "logps/chosen": -682.1751802884615, | |
| "logps/rejected": -270.47216796875, | |
| "loss": 0.2777, | |
| "rewards/chosen": -0.24122038254371056, | |
| "rewards/margins": 2.748665248816795, | |
| "rewards/rejected": -2.989885631360506, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.12289052358286456, | |
| "grad_norm": 24.75, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -8335642.105263158, | |
| "logits/rejected": -9525784.615384616, | |
| "logps/chosen": -303.1348170230263, | |
| "logps/rejected": -280.5024601862981, | |
| "loss": 0.3256, | |
| "rewards/chosen": 0.11083748466090153, | |
| "rewards/margins": 2.885276489412254, | |
| "rewards/rejected": -2.7744390047513523, | |
| "step": 71 | |
| }, | |
| { | |
| "epoch": 0.12462137602769364, | |
| "grad_norm": 12.625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -8330821.333333333, | |
| "logits/rejected": -3045587.4, | |
| "logps/chosen": -137.8349812825521, | |
| "logps/rejected": -273.801513671875, | |
| "loss": 0.1949, | |
| "rewards/chosen": 0.5404347976048788, | |
| "rewards/margins": 3.689098318417867, | |
| "rewards/rejected": -3.1486635208129883, | |
| "step": 72 | |
| }, | |
| { | |
| "epoch": 0.12635222847252273, | |
| "grad_norm": 21.75, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 9142004.444444444, | |
| "logits/rejected": -3207848.8571428573, | |
| "logps/chosen": -188.33430989583334, | |
| "logps/rejected": -207.32901436941964, | |
| "loss": 0.3074, | |
| "rewards/chosen": 0.039681686295403376, | |
| "rewards/margins": 3.252643155673194, | |
| "rewards/rejected": -3.2129614693777904, | |
| "step": 73 | |
| }, | |
| { | |
| "epoch": 0.1280830809173518, | |
| "grad_norm": 22.25, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -6030457.263157895, | |
| "logits/rejected": -3718584.0, | |
| "logps/chosen": -238.16568153782896, | |
| "logps/rejected": -195.5281700721154, | |
| "loss": 0.3596, | |
| "rewards/chosen": 0.006416631372351395, | |
| "rewards/margins": 2.2222877086898096, | |
| "rewards/rejected": -2.215871077317458, | |
| "step": 74 | |
| }, | |
| { | |
| "epoch": 0.12981393336218086, | |
| "grad_norm": 19.25, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -3802380.705882353, | |
| "logits/rejected": 7472531.2, | |
| "logps/chosen": -168.80958467371323, | |
| "logps/rejected": -285.90989583333334, | |
| "loss": 0.3219, | |
| "rewards/chosen": -0.224185635061825, | |
| "rewards/margins": 2.7274849480273673, | |
| "rewards/rejected": -2.9516705830891925, | |
| "step": 75 | |
| }, | |
| { | |
| "epoch": 0.13154478580700996, | |
| "grad_norm": 21.25, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -4579526.5, | |
| "logits/rejected": 6935348.0, | |
| "logps/chosen": -246.0178985595703, | |
| "logps/rejected": -326.98834228515625, | |
| "loss": 0.2722, | |
| "rewards/chosen": 0.10884374380111694, | |
| "rewards/margins": 3.900286853313446, | |
| "rewards/rejected": -3.791443109512329, | |
| "step": 76 | |
| }, | |
| { | |
| "epoch": 0.13327563825183902, | |
| "grad_norm": 23.875, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 2948165.6, | |
| "logits/rejected": -10498065.88235294, | |
| "logps/chosen": -323.5631510416667, | |
| "logps/rejected": -268.2412683823529, | |
| "loss": 0.2814, | |
| "rewards/chosen": -0.016830217838287354, | |
| "rewards/margins": 3.3728826207273146, | |
| "rewards/rejected": -3.389712838565602, | |
| "step": 77 | |
| }, | |
| { | |
| "epoch": 0.13500649069666812, | |
| "grad_norm": 19.75, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 3724001.263157895, | |
| "logits/rejected": -4905061.230769231, | |
| "logps/chosen": -196.80241313733552, | |
| "logps/rejected": -246.7076697716346, | |
| "loss": 0.3493, | |
| "rewards/chosen": 0.13470386203966642, | |
| "rewards/margins": 2.7459867975489813, | |
| "rewards/rejected": -2.611282935509315, | |
| "step": 78 | |
| }, | |
| { | |
| "epoch": 0.1367373431414972, | |
| "grad_norm": 19.125, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 882787.5, | |
| "logits/rejected": -2752756.8333333335, | |
| "logps/chosen": -152.87762451171875, | |
| "logps/rejected": -143.17495727539062, | |
| "loss": 0.3448, | |
| "rewards/chosen": -0.04928714632987976, | |
| "rewards/margins": 3.1339206834634146, | |
| "rewards/rejected": -3.1832078297932944, | |
| "step": 79 | |
| }, | |
| { | |
| "epoch": 0.13846819558632625, | |
| "grad_norm": 16.875, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -10046429.6, | |
| "logits/rejected": -9446046.545454545, | |
| "logps/chosen": -211.123828125, | |
| "logps/rejected": -224.4654873934659, | |
| "loss": 0.2231, | |
| "rewards/chosen": -0.09817437529563904, | |
| "rewards/margins": 3.1476648303595454, | |
| "rewards/rejected": -3.2458392056551846, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.14019904803115535, | |
| "grad_norm": 21.0, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 1529026.8235294118, | |
| "logits/rejected": -18009732.266666666, | |
| "logps/chosen": -215.9851505055147, | |
| "logps/rejected": -285.50341796875, | |
| "loss": 0.3038, | |
| "rewards/chosen": 0.07493850062875186, | |
| "rewards/margins": 3.3579501278260175, | |
| "rewards/rejected": -3.283011627197266, | |
| "step": 81 | |
| }, | |
| { | |
| "epoch": 0.14192990047598442, | |
| "grad_norm": 19.0, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 2760504.3076923075, | |
| "logits/rejected": -6295609.684210527, | |
| "logps/chosen": -247.9722618689904, | |
| "logps/rejected": -289.9179173519737, | |
| "loss": 0.2071, | |
| "rewards/chosen": 0.2483532978938176, | |
| "rewards/margins": 3.7582735185198453, | |
| "rewards/rejected": -3.509920220626028, | |
| "step": 82 | |
| }, | |
| { | |
| "epoch": 0.1436607529208135, | |
| "grad_norm": 18.125, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -12023732.923076924, | |
| "logits/rejected": -5987361.263157895, | |
| "logps/chosen": -222.31482872596155, | |
| "logps/rejected": -229.85079152960526, | |
| "loss": 0.2564, | |
| "rewards/chosen": -0.01725879082312951, | |
| "rewards/margins": 3.232607895546114, | |
| "rewards/rejected": -3.2498666863692436, | |
| "step": 83 | |
| }, | |
| { | |
| "epoch": 0.14539160536564258, | |
| "grad_norm": 17.75, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 612477.6842105263, | |
| "logits/rejected": 610641.0769230769, | |
| "logps/chosen": -134.88838918585526, | |
| "logps/rejected": -210.28667743389423, | |
| "loss": 0.3361, | |
| "rewards/chosen": -0.02787588772020842, | |
| "rewards/margins": 2.743228191306234, | |
| "rewards/rejected": -2.7711040790264425, | |
| "step": 84 | |
| }, | |
| { | |
| "epoch": 0.14712245781047165, | |
| "grad_norm": 19.75, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -8578859.294117646, | |
| "logits/rejected": -4098941.8666666667, | |
| "logps/chosen": -180.28768382352942, | |
| "logps/rejected": -149.85576171875, | |
| "loss": 0.3247, | |
| "rewards/chosen": 0.19388238121481502, | |
| "rewards/margins": 2.1001341614068725, | |
| "rewards/rejected": -1.9062517801920573, | |
| "step": 85 | |
| }, | |
| { | |
| "epoch": 0.14885331025530074, | |
| "grad_norm": 20.0, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 1573394.5882352942, | |
| "logits/rejected": 10871098.666666666, | |
| "logps/chosen": -191.07115981158088, | |
| "logps/rejected": -244.60188802083334, | |
| "loss": 0.3206, | |
| "rewards/chosen": 0.2755900551291073, | |
| "rewards/margins": 2.1674577563416726, | |
| "rewards/rejected": -1.891867701212565, | |
| "step": 86 | |
| }, | |
| { | |
| "epoch": 0.1505841627001298, | |
| "grad_norm": 21.125, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 1688797.142857143, | |
| "logits/rejected": -1483586.6666666667, | |
| "logps/chosen": -255.131103515625, | |
| "logps/rejected": -332.4247233072917, | |
| "loss": 0.2465, | |
| "rewards/chosen": 0.21271177700587682, | |
| "rewards/margins": 3.036478909235152, | |
| "rewards/rejected": -2.8237671322292752, | |
| "step": 87 | |
| }, | |
| { | |
| "epoch": 0.1523150151449589, | |
| "grad_norm": 19.375, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -15565679.0, | |
| "logits/rejected": -2823755.25, | |
| "logps/chosen": -226.8282012939453, | |
| "logps/rejected": -287.0397644042969, | |
| "loss": 0.2921, | |
| "rewards/chosen": 0.2816123962402344, | |
| "rewards/margins": 2.9265496730804443, | |
| "rewards/rejected": -2.64493727684021, | |
| "step": 88 | |
| }, | |
| { | |
| "epoch": 0.15404586758978797, | |
| "grad_norm": 20.125, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -413132.4, | |
| "logits/rejected": -10101343.05882353, | |
| "logps/chosen": -230.32145182291666, | |
| "logps/rejected": -227.8623764935662, | |
| "loss": 0.3061, | |
| "rewards/chosen": 0.1068873405456543, | |
| "rewards/margins": 2.7386797456180347, | |
| "rewards/rejected": -2.6317924050723804, | |
| "step": 89 | |
| }, | |
| { | |
| "epoch": 0.15577672003461704, | |
| "grad_norm": 16.375, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -2141461.846153846, | |
| "logits/rejected": -5844247.157894737, | |
| "logps/chosen": -115.6215350811298, | |
| "logps/rejected": -274.47286184210526, | |
| "loss": 0.2861, | |
| "rewards/chosen": -0.02882493459261381, | |
| "rewards/margins": 2.667185855780536, | |
| "rewards/rejected": -2.69601079037315, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.15750757247944613, | |
| "grad_norm": 25.125, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -12380478.545454545, | |
| "logits/rejected": -8278526.4, | |
| "logps/chosen": -197.9210759943182, | |
| "logps/rejected": -223.8676513671875, | |
| "loss": 0.3626, | |
| "rewards/chosen": 0.14342746951363303, | |
| "rewards/margins": 3.160957529328086, | |
| "rewards/rejected": -3.017530059814453, | |
| "step": 91 | |
| }, | |
| { | |
| "epoch": 0.1592384249242752, | |
| "grad_norm": 20.875, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -13449718.588235294, | |
| "logits/rejected": -4683606.933333334, | |
| "logps/chosen": -317.87795840992646, | |
| "logps/rejected": -267.261279296875, | |
| "loss": 0.2726, | |
| "rewards/chosen": 0.3811823059530819, | |
| "rewards/margins": 3.948626662235634, | |
| "rewards/rejected": -3.567444356282552, | |
| "step": 92 | |
| }, | |
| { | |
| "epoch": 0.1609692773691043, | |
| "grad_norm": 16.375, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -543169.375, | |
| "logits/rejected": -1371199.125, | |
| "logps/chosen": -150.7501678466797, | |
| "logps/rejected": -181.0672149658203, | |
| "loss": 0.3291, | |
| "rewards/chosen": -0.053351566195487976, | |
| "rewards/margins": 2.6354714184999466, | |
| "rewards/rejected": -2.6888229846954346, | |
| "step": 93 | |
| }, | |
| { | |
| "epoch": 0.16270012981393336, | |
| "grad_norm": 18.75, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 2523244.0, | |
| "logits/rejected": 2864016.380952381, | |
| "logps/chosen": -234.7637606534091, | |
| "logps/rejected": -178.11604817708334, | |
| "loss": 0.2795, | |
| "rewards/chosen": -0.16940477761355313, | |
| "rewards/margins": 2.3981288062545643, | |
| "rewards/rejected": -2.5675335838681175, | |
| "step": 94 | |
| }, | |
| { | |
| "epoch": 0.16443098225876243, | |
| "grad_norm": 21.625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 3101979.2, | |
| "logits/rejected": -5256099.0, | |
| "logps/chosen": -156.21822509765624, | |
| "logps/rejected": -249.0777791341146, | |
| "loss": 0.3557, | |
| "rewards/chosen": -0.0824066936969757, | |
| "rewards/margins": 3.3445211907227836, | |
| "rewards/rejected": -3.4269278844197593, | |
| "step": 95 | |
| }, | |
| { | |
| "epoch": 0.16616183470359153, | |
| "grad_norm": 16.75, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 7612368.7272727275, | |
| "logits/rejected": -5602336.761904762, | |
| "logps/chosen": -176.8388338955966, | |
| "logps/rejected": -257.1624348958333, | |
| "loss": 0.2211, | |
| "rewards/chosen": 0.0035542053255167875, | |
| "rewards/margins": 3.556209743248694, | |
| "rewards/rejected": -3.5526555379231772, | |
| "step": 96 | |
| }, | |
| { | |
| "epoch": 0.1678926871484206, | |
| "grad_norm": 19.125, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -1705522.0, | |
| "logits/rejected": -19220438.0, | |
| "logps/chosen": -229.12918090820312, | |
| "logps/rejected": -301.3127746582031, | |
| "loss": 0.2716, | |
| "rewards/chosen": 0.30286985635757446, | |
| "rewards/margins": 3.395688831806183, | |
| "rewards/rejected": -3.0928189754486084, | |
| "step": 97 | |
| }, | |
| { | |
| "epoch": 0.1696235395932497, | |
| "grad_norm": 24.0, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 12111708.307692308, | |
| "logits/rejected": -993389.2631578947, | |
| "logps/chosen": -285.0191180889423, | |
| "logps/rejected": -165.50485711348685, | |
| "loss": 0.2834, | |
| "rewards/chosen": 0.09072128626016471, | |
| "rewards/margins": 2.5426856144237133, | |
| "rewards/rejected": -2.4519643281635486, | |
| "step": 98 | |
| }, | |
| { | |
| "epoch": 0.17135439203807876, | |
| "grad_norm": 22.375, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -6123332.705882353, | |
| "logits/rejected": -4620013.866666666, | |
| "logps/chosen": -249.7477596507353, | |
| "logps/rejected": -251.340185546875, | |
| "loss": 0.3179, | |
| "rewards/chosen": 0.006986297228757073, | |
| "rewards/margins": 2.8065402319618302, | |
| "rewards/rejected": -2.799553934733073, | |
| "step": 99 | |
| }, | |
| { | |
| "epoch": 0.17308524448290782, | |
| "grad_norm": 17.625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -3662578.0, | |
| "logits/rejected": -3268372.75, | |
| "logps/chosen": -159.33383178710938, | |
| "logps/rejected": -199.64552307128906, | |
| "loss": 0.3467, | |
| "rewards/chosen": -0.1916092038154602, | |
| "rewards/margins": 2.2172593474388123, | |
| "rewards/rejected": -2.4088685512542725, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.17481609692773692, | |
| "grad_norm": 19.125, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -11227556.0, | |
| "logits/rejected": -4947570.4, | |
| "logps/chosen": -226.8167521158854, | |
| "logps/rejected": -230.9011474609375, | |
| "loss": 0.2496, | |
| "rewards/chosen": 0.0410018265247345, | |
| "rewards/margins": 3.1503684341907503, | |
| "rewards/rejected": -3.109366607666016, | |
| "step": 101 | |
| }, | |
| { | |
| "epoch": 0.176546949372566, | |
| "grad_norm": 21.0, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -4914299.368421053, | |
| "logits/rejected": -4945337.846153846, | |
| "logps/chosen": -197.87137643914474, | |
| "logps/rejected": -258.1477614182692, | |
| "loss": 0.3485, | |
| "rewards/chosen": -0.031160028357254833, | |
| "rewards/margins": 3.5723934501771506, | |
| "rewards/rejected": -3.603553478534405, | |
| "step": 102 | |
| }, | |
| { | |
| "epoch": 0.17827780181739505, | |
| "grad_norm": 20.375, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -7837846.222222222, | |
| "logits/rejected": -11910576.0, | |
| "logps/chosen": -174.4716796875, | |
| "logps/rejected": -307.66469029017856, | |
| "loss": 0.3231, | |
| "rewards/chosen": -0.013415685130490197, | |
| "rewards/margins": 2.68944691819331, | |
| "rewards/rejected": -2.7028626033238004, | |
| "step": 103 | |
| }, | |
| { | |
| "epoch": 0.18000865426222415, | |
| "grad_norm": 20.125, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -2126388.0, | |
| "logits/rejected": 277817.7894736842, | |
| "logps/chosen": -284.61337515024036, | |
| "logps/rejected": -307.56527549342104, | |
| "loss": 0.2237, | |
| "rewards/chosen": 0.27964045451237607, | |
| "rewards/margins": 3.5338759731184615, | |
| "rewards/rejected": -3.2542355186060856, | |
| "step": 104 | |
| }, | |
| { | |
| "epoch": 0.18173950670705322, | |
| "grad_norm": 21.25, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -1334305.875, | |
| "logits/rejected": -5412106.0, | |
| "logps/chosen": -228.7758331298828, | |
| "logps/rejected": -258.7738342285156, | |
| "loss": 0.3184, | |
| "rewards/chosen": -0.1122078001499176, | |
| "rewards/margins": 2.8912404477596283, | |
| "rewards/rejected": -3.003448247909546, | |
| "step": 105 | |
| }, | |
| { | |
| "epoch": 0.1834703591518823, | |
| "grad_norm": 18.25, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -10178373.05263158, | |
| "logits/rejected": -8054584.615384615, | |
| "logps/chosen": -173.59982781661185, | |
| "logps/rejected": -278.5968674879808, | |
| "loss": 0.3082, | |
| "rewards/chosen": 0.23594951629638672, | |
| "rewards/margins": 3.9558092997624326, | |
| "rewards/rejected": -3.719859783466046, | |
| "step": 106 | |
| }, | |
| { | |
| "epoch": 0.18520121159671138, | |
| "grad_norm": 20.0, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -7455461.647058823, | |
| "logits/rejected": -10272849.066666666, | |
| "logps/chosen": -190.27541934742646, | |
| "logps/rejected": -201.00078125, | |
| "loss": 0.3043, | |
| "rewards/chosen": 0.15676203896017635, | |
| "rewards/margins": 2.838399357889213, | |
| "rewards/rejected": -2.6816373189290363, | |
| "step": 107 | |
| }, | |
| { | |
| "epoch": 0.18693206404154045, | |
| "grad_norm": 21.625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -8889729.6, | |
| "logits/rejected": -3655384.0, | |
| "logps/chosen": -190.6529541015625, | |
| "logps/rejected": -243.38692220052084, | |
| "loss": 0.3275, | |
| "rewards/chosen": 0.024685271084308624, | |
| "rewards/margins": 3.879149484137694, | |
| "rewards/rejected": -3.8544642130533853, | |
| "step": 108 | |
| }, | |
| { | |
| "epoch": 0.18866291648636954, | |
| "grad_norm": 24.0, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -1406335.5789473683, | |
| "logits/rejected": -5790012.307692308, | |
| "logps/chosen": -160.40576171875, | |
| "logps/rejected": -207.287353515625, | |
| "loss": 0.3181, | |
| "rewards/chosen": -0.008714937850048668, | |
| "rewards/margins": 3.360927466559507, | |
| "rewards/rejected": -3.3696424044095554, | |
| "step": 109 | |
| }, | |
| { | |
| "epoch": 0.1903937689311986, | |
| "grad_norm": 17.25, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 1734898.1333333333, | |
| "logits/rejected": -7056189.176470588, | |
| "logps/chosen": -146.405810546875, | |
| "logps/rejected": -183.34127987132354, | |
| "loss": 0.3017, | |
| "rewards/chosen": 0.023581977685292563, | |
| "rewards/margins": 2.700872501438739, | |
| "rewards/rejected": -2.6772905237534466, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.1921246213760277, | |
| "grad_norm": 18.0, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 531886.4615384615, | |
| "logits/rejected": -6064568.421052632, | |
| "logps/chosen": -133.6158728966346, | |
| "logps/rejected": -266.82632606907896, | |
| "loss": 0.2827, | |
| "rewards/chosen": -0.16753161870516264, | |
| "rewards/margins": 2.9463925081708653, | |
| "rewards/rejected": -3.113924126876028, | |
| "step": 111 | |
| }, | |
| { | |
| "epoch": 0.19385547382085677, | |
| "grad_norm": 15.9375, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -7024642.4, | |
| "logits/rejected": -11717272.727272727, | |
| "logps/chosen": -179.82689208984374, | |
| "logps/rejected": -256.05178000710225, | |
| "loss": 0.1916, | |
| "rewards/chosen": 0.08213082551956177, | |
| "rewards/margins": 3.862654645876451, | |
| "rewards/rejected": -3.780523820356889, | |
| "step": 112 | |
| }, | |
| { | |
| "epoch": 0.19558632626568584, | |
| "grad_norm": 18.625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -7312199.5, | |
| "logits/rejected": -9691646.0, | |
| "logps/chosen": -190.84982299804688, | |
| "logps/rejected": -189.02178955078125, | |
| "loss": 0.2895, | |
| "rewards/chosen": -0.05902346968650818, | |
| "rewards/margins": 3.4700850546360016, | |
| "rewards/rejected": -3.5291085243225098, | |
| "step": 113 | |
| }, | |
| { | |
| "epoch": 0.19731717871051493, | |
| "grad_norm": 17.5, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -8977872.0, | |
| "logits/rejected": -2104818.5, | |
| "logps/chosen": -153.08140563964844, | |
| "logps/rejected": -362.2707824707031, | |
| "loss": 0.2608, | |
| "rewards/chosen": 0.1350196748971939, | |
| "rewards/margins": 4.345773592591286, | |
| "rewards/rejected": -4.210753917694092, | |
| "step": 114 | |
| }, | |
| { | |
| "epoch": 0.199048031155344, | |
| "grad_norm": 20.75, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -15021820.444444444, | |
| "logits/rejected": 3075554.8571428573, | |
| "logps/chosen": -254.52723524305554, | |
| "logps/rejected": -324.88724190848217, | |
| "loss": 0.2895, | |
| "rewards/chosen": 0.18979620933532715, | |
| "rewards/margins": 4.077897787094116, | |
| "rewards/rejected": -3.888101577758789, | |
| "step": 115 | |
| }, | |
| { | |
| "epoch": 0.2007788836001731, | |
| "grad_norm": 19.625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -8322480.0, | |
| "logits/rejected": 2723888.4210526315, | |
| "logps/chosen": -225.67595027043268, | |
| "logps/rejected": -165.18586811266448, | |
| "loss": 0.2731, | |
| "rewards/chosen": 0.10532364478478065, | |
| "rewards/margins": 3.168428977008773, | |
| "rewards/rejected": -3.0631053322239925, | |
| "step": 116 | |
| }, | |
| { | |
| "epoch": 0.20250973604500216, | |
| "grad_norm": 13.4375, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -3385337.6, | |
| "logits/rejected": -4593669.454545454, | |
| "logps/chosen": -155.44390869140625, | |
| "logps/rejected": -335.2124689275568, | |
| "loss": 0.2003, | |
| "rewards/chosen": -0.034807294607162476, | |
| "rewards/margins": 4.202970068563115, | |
| "rewards/rejected": -4.237777363170277, | |
| "step": 117 | |
| }, | |
| { | |
| "epoch": 0.20424058848983123, | |
| "grad_norm": 18.375, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -2332099.8181818184, | |
| "logits/rejected": -2151819.8, | |
| "logps/chosen": -175.2345525568182, | |
| "logps/rejected": -296.2167724609375, | |
| "loss": 0.3551, | |
| "rewards/chosen": 0.0834602876143022, | |
| "rewards/margins": 4.01979642347856, | |
| "rewards/rejected": -3.936336135864258, | |
| "step": 118 | |
| }, | |
| { | |
| "epoch": 0.20597144093466033, | |
| "grad_norm": 19.25, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -1813728.4210526317, | |
| "logits/rejected": -1663898.1538461538, | |
| "logps/chosen": -162.3670076069079, | |
| "logps/rejected": -307.88326322115387, | |
| "loss": 0.2952, | |
| "rewards/chosen": 0.22818475020559212, | |
| "rewards/margins": 3.5458337111994322, | |
| "rewards/rejected": -3.31764896099384, | |
| "step": 119 | |
| }, | |
| { | |
| "epoch": 0.2077022933794894, | |
| "grad_norm": 17.0, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 5888752.533333333, | |
| "logits/rejected": -2605564.705882353, | |
| "logps/chosen": -156.10784505208332, | |
| "logps/rejected": -253.71030560661765, | |
| "loss": 0.3016, | |
| "rewards/chosen": -0.10760652224222819, | |
| "rewards/margins": 3.660627281899546, | |
| "rewards/rejected": -3.768233804141774, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.2094331458243185, | |
| "grad_norm": 15.625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -7869675.076923077, | |
| "logits/rejected": -4793528.842105263, | |
| "logps/chosen": -159.2930626502404, | |
| "logps/rejected": -251.59606291118422, | |
| "loss": 0.2563, | |
| "rewards/chosen": -0.1771384019118089, | |
| "rewards/margins": 3.1404216936242726, | |
| "rewards/rejected": -3.3175600955360816, | |
| "step": 121 | |
| }, | |
| { | |
| "epoch": 0.21116399826914756, | |
| "grad_norm": 18.125, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -3852856.0, | |
| "logits/rejected": 1017586.2352941176, | |
| "logps/chosen": -130.758642578125, | |
| "logps/rejected": -310.29397403492646, | |
| "loss": 0.2913, | |
| "rewards/chosen": -0.17760810852050782, | |
| "rewards/margins": 3.5648737963508155, | |
| "rewards/rejected": -3.7424819048713234, | |
| "step": 122 | |
| }, | |
| { | |
| "epoch": 0.21289485071397662, | |
| "grad_norm": 21.875, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 394461.8947368421, | |
| "logits/rejected": -11013158.153846154, | |
| "logps/chosen": -211.87534693667763, | |
| "logps/rejected": -266.10584435096155, | |
| "loss": 0.3519, | |
| "rewards/chosen": -0.09623796061465614, | |
| "rewards/margins": 3.0594592017200792, | |
| "rewards/rejected": -3.1556971623347354, | |
| "step": 123 | |
| }, | |
| { | |
| "epoch": 0.21462570315880572, | |
| "grad_norm": 20.125, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -375859.125, | |
| "logits/rejected": -18902716.0, | |
| "logps/chosen": -144.1176513671875, | |
| "logps/rejected": -328.43532307942706, | |
| "loss": 0.3587, | |
| "rewards/chosen": -0.10335218906402588, | |
| "rewards/margins": 3.1057602961858115, | |
| "rewards/rejected": -3.2091124852498374, | |
| "step": 124 | |
| }, | |
| { | |
| "epoch": 0.2163565556036348, | |
| "grad_norm": 19.375, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -9888419.2, | |
| "logits/rejected": -13417634.823529411, | |
| "logps/chosen": -207.53665364583333, | |
| "logps/rejected": -305.8712373621324, | |
| "loss": 0.2801, | |
| "rewards/chosen": 0.16173944473266602, | |
| "rewards/margins": 3.5778553738313565, | |
| "rewards/rejected": -3.4161159290986904, | |
| "step": 125 | |
| }, | |
| { | |
| "epoch": 0.21808740804846388, | |
| "grad_norm": 15.625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -12050750.4, | |
| "logits/rejected": -13769786.181818182, | |
| "logps/chosen": -183.52490234375, | |
| "logps/rejected": -278.15651633522725, | |
| "loss": 0.2183, | |
| "rewards/chosen": 0.14027655124664307, | |
| "rewards/margins": 3.1395226283506914, | |
| "rewards/rejected": -2.9992460771040483, | |
| "step": 126 | |
| }, | |
| { | |
| "epoch": 0.21981826049329295, | |
| "grad_norm": 17.25, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -12270530.666666666, | |
| "logits/rejected": -8869286.4, | |
| "logps/chosen": -190.94978841145834, | |
| "logps/rejected": -284.20654296875, | |
| "loss": 0.233, | |
| "rewards/chosen": 0.16113528609275818, | |
| "rewards/margins": 3.4381788194179537, | |
| "rewards/rejected": -3.2770435333251955, | |
| "step": 127 | |
| }, | |
| { | |
| "epoch": 0.22154911293812202, | |
| "grad_norm": 19.0, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -15397617.066666666, | |
| "logits/rejected": -13504640.94117647, | |
| "logps/chosen": -223.714453125, | |
| "logps/rejected": -284.49543313419116, | |
| "loss": 0.2465, | |
| "rewards/chosen": 0.2658435821533203, | |
| "rewards/margins": 3.566493337294635, | |
| "rewards/rejected": -3.3006497551413143, | |
| "step": 128 | |
| }, | |
| { | |
| "epoch": 0.2232799653829511, | |
| "grad_norm": 18.625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -13344241.333333334, | |
| "logits/rejected": 3126395.2, | |
| "logps/chosen": -236.7048543294271, | |
| "logps/rejected": -158.71007080078124, | |
| "loss": 0.2397, | |
| "rewards/chosen": 0.05072679618994395, | |
| "rewards/margins": 3.053556347886721, | |
| "rewards/rejected": -3.0028295516967773, | |
| "step": 129 | |
| }, | |
| { | |
| "epoch": 0.22501081782778018, | |
| "grad_norm": 18.75, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -1188826.4, | |
| "logits/rejected": -7702007.529411765, | |
| "logps/chosen": -194.06878255208332, | |
| "logps/rejected": -285.5570714613971, | |
| "loss": 0.287, | |
| "rewards/chosen": -0.050044012069702146, | |
| "rewards/margins": 2.950371789932251, | |
| "rewards/rejected": -3.000415802001953, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.22674167027260925, | |
| "grad_norm": 24.25, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -9090264.421052631, | |
| "logits/rejected": -13554491.076923076, | |
| "logps/chosen": -228.48311574835526, | |
| "logps/rejected": -317.5834209735577, | |
| "loss": 0.3322, | |
| "rewards/chosen": 0.17829758242556923, | |
| "rewards/margins": 2.7610747524601247, | |
| "rewards/rejected": -2.5827771700345554, | |
| "step": 131 | |
| }, | |
| { | |
| "epoch": 0.22847252271743834, | |
| "grad_norm": 20.875, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -11592303.111111112, | |
| "logits/rejected": -3384686.285714286, | |
| "logps/chosen": -236.07706705729166, | |
| "logps/rejected": -231.01377650669642, | |
| "loss": 0.3204, | |
| "rewards/chosen": 0.08443025747934978, | |
| "rewards/margins": 3.5145191181273687, | |
| "rewards/rejected": -3.430088860648019, | |
| "step": 132 | |
| }, | |
| { | |
| "epoch": 0.2302033751622674, | |
| "grad_norm": 18.5, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -4109541.0, | |
| "logits/rejected": -10256295.0, | |
| "logps/chosen": -150.5631561279297, | |
| "logps/rejected": -251.85006713867188, | |
| "loss": 0.3414, | |
| "rewards/chosen": -0.03023519366979599, | |
| "rewards/margins": 2.841710902750492, | |
| "rewards/rejected": -2.871946096420288, | |
| "step": 133 | |
| }, | |
| { | |
| "epoch": 0.2319342276070965, | |
| "grad_norm": 19.875, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -17335990.153846152, | |
| "logits/rejected": -11810266.94736842, | |
| "logps/chosen": -255.87794846754807, | |
| "logps/rejected": -292.57041529605266, | |
| "loss": 0.2403, | |
| "rewards/chosen": -0.009397160548430223, | |
| "rewards/margins": 3.8956148698021043, | |
| "rewards/rejected": -3.9050120303505347, | |
| "step": 134 | |
| }, | |
| { | |
| "epoch": 0.23366508005192557, | |
| "grad_norm": 19.0, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 3816658.933333333, | |
| "logits/rejected": -21506733.17647059, | |
| "logps/chosen": -192.03406575520833, | |
| "logps/rejected": -326.34670840992646, | |
| "loss": 0.2583, | |
| "rewards/chosen": -0.08073126475016276, | |
| "rewards/margins": 4.162721368378285, | |
| "rewards/rejected": -4.243452633128447, | |
| "step": 135 | |
| }, | |
| { | |
| "epoch": 0.23539593249675464, | |
| "grad_norm": 23.375, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -5209688.421052632, | |
| "logits/rejected": 4241887.076923077, | |
| "logps/chosen": -241.80124383223685, | |
| "logps/rejected": -267.38955453725964, | |
| "loss": 0.3287, | |
| "rewards/chosen": 0.1192607001254433, | |
| "rewards/margins": 3.0787412792082254, | |
| "rewards/rejected": -2.9594805790827823, | |
| "step": 136 | |
| }, | |
| { | |
| "epoch": 0.23712678494158373, | |
| "grad_norm": 16.625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -13466715.733333332, | |
| "logits/rejected": -6725747.764705882, | |
| "logps/chosen": -169.23546549479167, | |
| "logps/rejected": -301.47449448529414, | |
| "loss": 0.2489, | |
| "rewards/chosen": 0.17315847078959148, | |
| "rewards/margins": 4.150024351419187, | |
| "rewards/rejected": -3.9768658806295956, | |
| "step": 137 | |
| }, | |
| { | |
| "epoch": 0.2388576373864128, | |
| "grad_norm": 20.625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -7070817.5, | |
| "logits/rejected": -5078294.5, | |
| "logps/chosen": -224.52960205078125, | |
| "logps/rejected": -250.922119140625, | |
| "loss": 0.299, | |
| "rewards/chosen": -0.062333978712558746, | |
| "rewards/margins": 3.91761764138937, | |
| "rewards/rejected": -3.9799516201019287, | |
| "step": 138 | |
| }, | |
| { | |
| "epoch": 0.2405884898312419, | |
| "grad_norm": 20.5, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -10853539.555555556, | |
| "logits/rejected": -8516350.857142856, | |
| "logps/chosen": -245.61515299479166, | |
| "logps/rejected": -311.77559988839283, | |
| "loss": 0.2919, | |
| "rewards/chosen": 0.143819702996148, | |
| "rewards/margins": 4.062012430221316, | |
| "rewards/rejected": -3.9181927272251675, | |
| "step": 139 | |
| }, | |
| { | |
| "epoch": 0.24231934227607096, | |
| "grad_norm": 19.0, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -23256322.666666668, | |
| "logits/rejected": -14564356.8, | |
| "logps/chosen": -252.69757080078125, | |
| "logps/rejected": -324.65302734375, | |
| "loss": 0.1926, | |
| "rewards/chosen": 0.15854175885518393, | |
| "rewards/margins": 4.718555339177449, | |
| "rewards/rejected": -4.5600135803222654, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.24405019472090003, | |
| "grad_norm": 18.875, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -13540496.0, | |
| "logits/rejected": -14330663.111111112, | |
| "logps/chosen": -261.66441127232144, | |
| "logps/rejected": -279.0700954861111, | |
| "loss": 0.2759, | |
| "rewards/chosen": -0.16050028800964355, | |
| "rewards/margins": 3.2266637219323053, | |
| "rewards/rejected": -3.387164009941949, | |
| "step": 141 | |
| }, | |
| { | |
| "epoch": 0.24578104716572913, | |
| "grad_norm": 22.25, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -16567258.352941176, | |
| "logits/rejected": -9630573.866666667, | |
| "logps/chosen": -248.67431640625, | |
| "logps/rejected": -195.8955078125, | |
| "loss": 0.2962, | |
| "rewards/chosen": 0.18976323744829962, | |
| "rewards/margins": 3.4638149037080654, | |
| "rewards/rejected": -3.2740516662597656, | |
| "step": 142 | |
| }, | |
| { | |
| "epoch": 0.2475118996105582, | |
| "grad_norm": 16.75, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -12402342.153846154, | |
| "logits/rejected": -7839504.842105263, | |
| "logps/chosen": -210.70182917668268, | |
| "logps/rejected": -379.2096525493421, | |
| "loss": 0.2114, | |
| "rewards/chosen": 0.16687591259296125, | |
| "rewards/margins": 5.201281671099335, | |
| "rewards/rejected": -5.034405758506374, | |
| "step": 143 | |
| }, | |
| { | |
| "epoch": 0.2492427520553873, | |
| "grad_norm": 19.75, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -4255770.285714285, | |
| "logits/rejected": -3999491.111111111, | |
| "logps/chosen": -195.87995256696428, | |
| "logps/rejected": -214.72549099392361, | |
| "loss": 0.2982, | |
| "rewards/chosen": -0.21987019266401017, | |
| "rewards/margins": 2.6090391514793274, | |
| "rewards/rejected": -2.8289093441433377, | |
| "step": 144 | |
| }, | |
| { | |
| "epoch": 0.25097360450021633, | |
| "grad_norm": 22.375, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -4979902.666666667, | |
| "logits/rejected": -11014548.57142857, | |
| "logps/chosen": -255.74172634548611, | |
| "logps/rejected": -255.93235560825892, | |
| "loss": 0.3164, | |
| "rewards/chosen": -0.09737168418036567, | |
| "rewards/margins": 3.5333697076827764, | |
| "rewards/rejected": -3.630741391863142, | |
| "step": 145 | |
| }, | |
| { | |
| "epoch": 0.25270445694504545, | |
| "grad_norm": 17.75, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -7553311.5, | |
| "logits/rejected": -12855062.0, | |
| "logps/chosen": -181.3147735595703, | |
| "logps/rejected": -319.3401184082031, | |
| "loss": 0.2509, | |
| "rewards/chosen": 0.28162485361099243, | |
| "rewards/margins": 4.752773344516754, | |
| "rewards/rejected": -4.471148490905762, | |
| "step": 146 | |
| }, | |
| { | |
| "epoch": 0.2544353093898745, | |
| "grad_norm": 16.375, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -18317777.6, | |
| "logits/rejected": -8850414.545454545, | |
| "logps/chosen": -208.5265380859375, | |
| "logps/rejected": -253.0011319247159, | |
| "loss": 0.2135, | |
| "rewards/chosen": 0.22829954624176024, | |
| "rewards/margins": 3.561434630914168, | |
| "rewards/rejected": -3.3331350846724077, | |
| "step": 147 | |
| }, | |
| { | |
| "epoch": 0.2561661618347036, | |
| "grad_norm": 22.5, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 7776780.0, | |
| "logits/rejected": -13990768.0, | |
| "logps/chosen": -222.059326171875, | |
| "logps/rejected": -333.6552734375, | |
| "loss": 0.3596, | |
| "rewards/chosen": -0.05462043881416321, | |
| "rewards/margins": 2.7295163333415986, | |
| "rewards/rejected": -2.7841367721557617, | |
| "step": 148 | |
| }, | |
| { | |
| "epoch": 0.25789701427953265, | |
| "grad_norm": 20.5, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -501711.76470588235, | |
| "logits/rejected": -4720805.333333333, | |
| "logps/chosen": -165.69982192095588, | |
| "logps/rejected": -258.313623046875, | |
| "loss": 0.3126, | |
| "rewards/chosen": -0.00996632085126989, | |
| "rewards/margins": 2.9709932535302404, | |
| "rewards/rejected": -2.9809595743815103, | |
| "step": 149 | |
| }, | |
| { | |
| "epoch": 0.2596278667243617, | |
| "grad_norm": 16.0, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -11498374.153846154, | |
| "logits/rejected": -12492654.315789474, | |
| "logps/chosen": -136.2483191856971, | |
| "logps/rejected": -243.18302837171052, | |
| "loss": 0.27, | |
| "rewards/chosen": -0.3298172950744629, | |
| "rewards/margins": 2.9719581854970833, | |
| "rewards/rejected": -3.3017754805715462, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.26135871916919085, | |
| "grad_norm": 20.375, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -18241306.0, | |
| "logits/rejected": -5504833.5, | |
| "logps/chosen": -285.53369140625, | |
| "logps/rejected": -346.5982360839844, | |
| "loss": 0.2638, | |
| "rewards/chosen": 0.2393263578414917, | |
| "rewards/margins": 3.530009150505066, | |
| "rewards/rejected": -3.290682792663574, | |
| "step": 151 | |
| }, | |
| { | |
| "epoch": 0.2630895716140199, | |
| "grad_norm": 21.0, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -9077254.588235294, | |
| "logits/rejected": -16832766.933333334, | |
| "logps/chosen": -218.34030330882354, | |
| "logps/rejected": -284.46845703125, | |
| "loss": 0.3062, | |
| "rewards/chosen": 0.03087810558431289, | |
| "rewards/margins": 3.274915225131839, | |
| "rewards/rejected": -3.244037119547526, | |
| "step": 152 | |
| }, | |
| { | |
| "epoch": 0.264820424058849, | |
| "grad_norm": 17.375, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 3784820.0, | |
| "logits/rejected": -27976580.266666666, | |
| "logps/chosen": -176.8842342601103, | |
| "logps/rejected": -371.0309244791667, | |
| "loss": 0.2938, | |
| "rewards/chosen": 0.037597624694599825, | |
| "rewards/margins": 3.3108126006874383, | |
| "rewards/rejected": -3.2732149759928384, | |
| "step": 153 | |
| }, | |
| { | |
| "epoch": 0.26655127650367805, | |
| "grad_norm": 19.5, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 8110033.066666666, | |
| "logits/rejected": -6098681.411764706, | |
| "logps/chosen": -224.60833333333332, | |
| "logps/rejected": -270.91673368566177, | |
| "loss": 0.2667, | |
| "rewards/chosen": 0.19785807927449545, | |
| "rewards/margins": 3.2095348676045736, | |
| "rewards/rejected": -3.011676788330078, | |
| "step": 154 | |
| }, | |
| { | |
| "epoch": 0.2682821289485071, | |
| "grad_norm": 24.375, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -10915676.16, | |
| "logits/rejected": -12048051.42857143, | |
| "logps/chosen": -200.02255859375, | |
| "logps/rejected": -363.83028738839283, | |
| "loss": 0.4141, | |
| "rewards/chosen": -0.20076709747314453, | |
| "rewards/margins": 4.246241580418179, | |
| "rewards/rejected": -4.447008677891323, | |
| "step": 155 | |
| }, | |
| { | |
| "epoch": 0.27001298139333624, | |
| "grad_norm": 21.0, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 4039928.5, | |
| "logits/rejected": -297019.3125, | |
| "logps/chosen": -122.26431274414062, | |
| "logps/rejected": -205.24542236328125, | |
| "loss": 0.2976, | |
| "rewards/chosen": 0.13761137425899506, | |
| "rewards/margins": 2.9543447345495224, | |
| "rewards/rejected": -2.8167333602905273, | |
| "step": 156 | |
| }, | |
| { | |
| "epoch": 0.2717438338381653, | |
| "grad_norm": 24.0, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -11950380.444444444, | |
| "logits/rejected": -14319524.57142857, | |
| "logps/chosen": -316.365234375, | |
| "logps/rejected": -238.67583356584822, | |
| "loss": 0.3578, | |
| "rewards/chosen": -0.22798464033338758, | |
| "rewards/margins": 2.9639355038839676, | |
| "rewards/rejected": -3.191920144217355, | |
| "step": 157 | |
| }, | |
| { | |
| "epoch": 0.2734746862829944, | |
| "grad_norm": 16.625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -7220196.923076923, | |
| "logits/rejected": -7274026.105263158, | |
| "logps/chosen": -168.70252403846155, | |
| "logps/rejected": -272.20877878289474, | |
| "loss": 0.2521, | |
| "rewards/chosen": 0.034194111824035645, | |
| "rewards/margins": 3.5594172916914286, | |
| "rewards/rejected": -3.525223179867393, | |
| "step": 158 | |
| }, | |
| { | |
| "epoch": 0.27520553872782344, | |
| "grad_norm": 19.625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -7790591.111111111, | |
| "logits/rejected": -4348877.714285715, | |
| "logps/chosen": -150.17240397135416, | |
| "logps/rejected": -257.23592703683033, | |
| "loss": 0.2867, | |
| "rewards/chosen": 0.2716523011525472, | |
| "rewards/margins": 3.2814045747121177, | |
| "rewards/rejected": -3.0097522735595703, | |
| "step": 159 | |
| }, | |
| { | |
| "epoch": 0.2769363911726525, | |
| "grad_norm": 21.5, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -6992461.0, | |
| "logits/rejected": -9803589.0, | |
| "logps/chosen": -197.62159729003906, | |
| "logps/rejected": -200.63839721679688, | |
| "loss": 0.3119, | |
| "rewards/chosen": 0.2022910714149475, | |
| "rewards/margins": 2.61891371011734, | |
| "rewards/rejected": -2.4166226387023926, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 0.27866724361748163, | |
| "grad_norm": 15.5625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 2316156.0, | |
| "logits/rejected": -6923803.294117647, | |
| "logps/chosen": -157.97198893229168, | |
| "logps/rejected": -226.3733340992647, | |
| "loss": 0.2941, | |
| "rewards/chosen": -0.181062380472819, | |
| "rewards/margins": 3.137106035269943, | |
| "rewards/rejected": -3.318168415742762, | |
| "step": 161 | |
| }, | |
| { | |
| "epoch": 0.2803980960623107, | |
| "grad_norm": 19.375, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 1817514.6666666667, | |
| "logits/rejected": -4328779.428571428, | |
| "logps/chosen": -125.52289496527777, | |
| "logps/rejected": -229.05801827566964, | |
| "loss": 0.3201, | |
| "rewards/chosen": 0.1051819192038642, | |
| "rewards/margins": 2.722233724972558, | |
| "rewards/rejected": -2.617051805768694, | |
| "step": 162 | |
| }, | |
| { | |
| "epoch": 0.28212894850713977, | |
| "grad_norm": 21.625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -7507249.411764706, | |
| "logits/rejected": 2358673.6, | |
| "logps/chosen": -203.8069278492647, | |
| "logps/rejected": -239.440380859375, | |
| "loss": 0.2975, | |
| "rewards/chosen": 0.04498655655804802, | |
| "rewards/margins": 3.3732679591459385, | |
| "rewards/rejected": -3.3282814025878906, | |
| "step": 163 | |
| }, | |
| { | |
| "epoch": 0.28385980095196883, | |
| "grad_norm": 18.5, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -17538291.2, | |
| "logits/rejected": -8524349.176470589, | |
| "logps/chosen": -203.443017578125, | |
| "logps/rejected": -309.54041245404414, | |
| "loss": 0.2556, | |
| "rewards/chosen": 0.23375027974446613, | |
| "rewards/margins": 3.705425703759287, | |
| "rewards/rejected": -3.471675424014821, | |
| "step": 164 | |
| }, | |
| { | |
| "epoch": 0.2855906533967979, | |
| "grad_norm": 15.5625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -11055366.857142856, | |
| "logits/rejected": -1121562.3333333333, | |
| "logps/chosen": -159.264892578125, | |
| "logps/rejected": -286.91015625, | |
| "loss": 0.2449, | |
| "rewards/chosen": 0.12227598258427211, | |
| "rewards/margins": 4.091070063530452, | |
| "rewards/rejected": -3.9687940809461804, | |
| "step": 165 | |
| }, | |
| { | |
| "epoch": 0.287321505841627, | |
| "grad_norm": 23.625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 657442.6, | |
| "logits/rejected": -9425391.333333334, | |
| "logps/chosen": -220.0943603515625, | |
| "logps/rejected": -196.21675618489584, | |
| "loss": 0.3849, | |
| "rewards/chosen": -0.32888593673706057, | |
| "rewards/margins": 2.8519148190816246, | |
| "rewards/rejected": -3.180800755818685, | |
| "step": 166 | |
| }, | |
| { | |
| "epoch": 0.2890523582864561, | |
| "grad_norm": 22.75, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -4699375.555555556, | |
| "logits/rejected": -3600353.1428571427, | |
| "logps/chosen": -265.4497341579861, | |
| "logps/rejected": -267.37051827566967, | |
| "loss": 0.2924, | |
| "rewards/chosen": 0.23302984237670898, | |
| "rewards/margins": 3.618199280330113, | |
| "rewards/rejected": -3.385169437953404, | |
| "step": 167 | |
| }, | |
| { | |
| "epoch": 0.29078321073128516, | |
| "grad_norm": 24.5, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -15453594.666666666, | |
| "logits/rejected": -7304598.5, | |
| "logps/chosen": -211.82305908203125, | |
| "logps/rejected": -305.252685546875, | |
| "loss": 0.4196, | |
| "rewards/chosen": -0.15757346153259277, | |
| "rewards/margins": 3.7481679916381836, | |
| "rewards/rejected": -3.9057414531707764, | |
| "step": 168 | |
| }, | |
| { | |
| "epoch": 0.2925140631761142, | |
| "grad_norm": 19.75, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 917019.2941176471, | |
| "logits/rejected": -11583381.333333334, | |
| "logps/chosen": -187.60915958180146, | |
| "logps/rejected": -256.85833333333335, | |
| "loss": 0.2921, | |
| "rewards/chosen": -0.004517814692328958, | |
| "rewards/margins": 3.6537013320361864, | |
| "rewards/rejected": -3.6582191467285154, | |
| "step": 169 | |
| }, | |
| { | |
| "epoch": 0.2942449156209433, | |
| "grad_norm": 20.75, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -5889556.705882353, | |
| "logits/rejected": -12821309.866666667, | |
| "logps/chosen": -256.27088120404414, | |
| "logps/rejected": -308.8744791666667, | |
| "loss": 0.2513, | |
| "rewards/chosen": 0.38170385360717773, | |
| "rewards/margins": 4.123530483245849, | |
| "rewards/rejected": -3.741826629638672, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 0.2959757680657724, | |
| "grad_norm": 17.875, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 16747204.0, | |
| "logits/rejected": 620209.6, | |
| "logps/chosen": -243.82991536458334, | |
| "logps/rejected": -253.942626953125, | |
| "loss": 0.2081, | |
| "rewards/chosen": 0.4542102813720703, | |
| "rewards/margins": 3.8508056640625, | |
| "rewards/rejected": -3.3965953826904296, | |
| "step": 171 | |
| }, | |
| { | |
| "epoch": 0.2977066205106015, | |
| "grad_norm": 17.375, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -12614680.615384616, | |
| "logits/rejected": -7096439.578947368, | |
| "logps/chosen": -130.17988469050482, | |
| "logps/rejected": -229.95723684210526, | |
| "loss": 0.2725, | |
| "rewards/chosen": 0.0035039232327387882, | |
| "rewards/margins": 2.607151255675173, | |
| "rewards/rejected": -2.603647332442434, | |
| "step": 172 | |
| }, | |
| { | |
| "epoch": 0.29943747295543055, | |
| "grad_norm": 16.5, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -3154612.0, | |
| "logits/rejected": -9195659.2, | |
| "logps/chosen": -173.6625773111979, | |
| "logps/rejected": -270.675537109375, | |
| "loss": 0.2182, | |
| "rewards/chosen": 0.14845428864161173, | |
| "rewards/margins": 3.5584659616152443, | |
| "rewards/rejected": -3.4100116729736327, | |
| "step": 173 | |
| }, | |
| { | |
| "epoch": 0.3011683254002596, | |
| "grad_norm": 19.625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -9196400.0, | |
| "logits/rejected": -15175718.0, | |
| "logps/chosen": -157.8928680419922, | |
| "logps/rejected": -259.74560546875, | |
| "loss": 0.3143, | |
| "rewards/chosen": -0.17662523686885834, | |
| "rewards/margins": 3.229040876030922, | |
| "rewards/rejected": -3.4056661128997803, | |
| "step": 174 | |
| }, | |
| { | |
| "epoch": 0.3028991778450887, | |
| "grad_norm": 18.875, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -15626394.352941176, | |
| "logits/rejected": -7392939.2, | |
| "logps/chosen": -241.25695082720588, | |
| "logps/rejected": -236.45672200520832, | |
| "loss": 0.2589, | |
| "rewards/chosen": 0.28153758890488567, | |
| "rewards/margins": 4.230858064165302, | |
| "rewards/rejected": -3.9493204752604165, | |
| "step": 175 | |
| }, | |
| { | |
| "epoch": 0.3046300302899178, | |
| "grad_norm": 22.125, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -9918681.6, | |
| "logits/rejected": -10010532.0, | |
| "logps/chosen": -202.339111328125, | |
| "logps/rejected": -184.87152099609375, | |
| "loss": 0.3503, | |
| "rewards/chosen": -0.07323684692382812, | |
| "rewards/margins": 3.2869134902954102, | |
| "rewards/rejected": -3.3601503372192383, | |
| "step": 176 | |
| }, | |
| { | |
| "epoch": 0.3063608827347469, | |
| "grad_norm": 17.75, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -9519585.0, | |
| "logits/rejected": -17195120.0, | |
| "logps/chosen": -151.671875, | |
| "logps/rejected": -259.06878662109375, | |
| "loss": 0.2859, | |
| "rewards/chosen": 0.010412598960101604, | |
| "rewards/margins": 3.337350751273334, | |
| "rewards/rejected": -3.3269381523132324, | |
| "step": 177 | |
| }, | |
| { | |
| "epoch": 0.30809173517957594, | |
| "grad_norm": 21.875, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -16214270.11764706, | |
| "logits/rejected": -9008196.266666668, | |
| "logps/chosen": -273.4817899816176, | |
| "logps/rejected": -351.15856119791664, | |
| "loss": 0.2819, | |
| "rewards/chosen": 0.03458939930971931, | |
| "rewards/margins": 3.560055555315579, | |
| "rewards/rejected": -3.5254661560058596, | |
| "step": 178 | |
| }, | |
| { | |
| "epoch": 0.309822587624405, | |
| "grad_norm": 18.625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -4070659.5555555555, | |
| "logits/rejected": 564305.2857142857, | |
| "logps/chosen": -159.47158474392361, | |
| "logps/rejected": -243.50922502790178, | |
| "loss": 0.3114, | |
| "rewards/chosen": 0.13283884525299072, | |
| "rewards/margins": 3.35727219922202, | |
| "rewards/rejected": -3.224433353969029, | |
| "step": 179 | |
| }, | |
| { | |
| "epoch": 0.3115534400692341, | |
| "grad_norm": 18.625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -6579336.888888889, | |
| "logits/rejected": -11798860.57142857, | |
| "logps/chosen": -194.06880696614584, | |
| "logps/rejected": -265.37472098214283, | |
| "loss": 0.3398, | |
| "rewards/chosen": -0.14026531908247206, | |
| "rewards/margins": 2.981163579320151, | |
| "rewards/rejected": -3.121428898402623, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 0.3132842925140632, | |
| "grad_norm": 17.625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -5290882.0, | |
| "logits/rejected": -21157156.0, | |
| "logps/chosen": -148.17227172851562, | |
| "logps/rejected": -296.1889953613281, | |
| "loss": 0.3044, | |
| "rewards/chosen": -0.1293964684009552, | |
| "rewards/margins": 3.0376605689525604, | |
| "rewards/rejected": -3.1670570373535156, | |
| "step": 181 | |
| }, | |
| { | |
| "epoch": 0.31501514495889227, | |
| "grad_norm": 21.375, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -9015065.263157895, | |
| "logits/rejected": -6190756.923076923, | |
| "logps/chosen": -208.37291837993422, | |
| "logps/rejected": -260.5161884014423, | |
| "loss": 0.3085, | |
| "rewards/chosen": 0.10386697869551809, | |
| "rewards/margins": 3.3943309397832584, | |
| "rewards/rejected": -3.2904639610877404, | |
| "step": 182 | |
| }, | |
| { | |
| "epoch": 0.31674599740372134, | |
| "grad_norm": 19.0, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -10990801.066666666, | |
| "logits/rejected": -6380602.352941177, | |
| "logps/chosen": -166.50836588541668, | |
| "logps/rejected": -261.99778837316177, | |
| "loss": 0.2828, | |
| "rewards/chosen": -0.09178520043690999, | |
| "rewards/margins": 3.105228430149602, | |
| "rewards/rejected": -3.197013630586512, | |
| "step": 183 | |
| }, | |
| { | |
| "epoch": 0.3184768498485504, | |
| "grad_norm": 22.625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -15325334.857142856, | |
| "logits/rejected": -24043242.181818184, | |
| "logps/chosen": -213.09461030505952, | |
| "logps/rejected": -345.95725319602275, | |
| "loss": 0.3397, | |
| "rewards/chosen": 0.22786199478876024, | |
| "rewards/margins": 2.5508950675204716, | |
| "rewards/rejected": -2.3230330727317114, | |
| "step": 184 | |
| }, | |
| { | |
| "epoch": 0.32020770229337947, | |
| "grad_norm": 21.25, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -11346512.94117647, | |
| "logits/rejected": -2746510.4, | |
| "logps/chosen": -264.31959443933823, | |
| "logps/rejected": -207.83736979166667, | |
| "loss": 0.2974, | |
| "rewards/chosen": 0.12100423083585851, | |
| "rewards/margins": 2.763645679810468, | |
| "rewards/rejected": -2.6426414489746093, | |
| "step": 185 | |
| }, | |
| { | |
| "epoch": 0.3219385547382086, | |
| "grad_norm": 19.125, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -10335120.0, | |
| "logits/rejected": -11680630.0, | |
| "logps/chosen": -169.1227264404297, | |
| "logps/rejected": -242.72560119628906, | |
| "loss": 0.3009, | |
| "rewards/chosen": 0.12381087243556976, | |
| "rewards/margins": 3.1314540952444077, | |
| "rewards/rejected": -3.007643222808838, | |
| "step": 186 | |
| }, | |
| { | |
| "epoch": 0.32366940718303766, | |
| "grad_norm": 18.375, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -15741749.714285715, | |
| "logits/rejected": -5845185.333333333, | |
| "logps/chosen": -198.52054268973214, | |
| "logps/rejected": -295.60541449652777, | |
| "loss": 0.2393, | |
| "rewards/chosen": 0.4524484021323068, | |
| "rewards/margins": 3.021870806103661, | |
| "rewards/rejected": -2.569422403971354, | |
| "step": 187 | |
| }, | |
| { | |
| "epoch": 0.32540025962786673, | |
| "grad_norm": 19.375, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -9898615.578947369, | |
| "logits/rejected": -2790789.846153846, | |
| "logps/chosen": -154.5710320723684, | |
| "logps/rejected": -181.50324894831732, | |
| "loss": 0.3752, | |
| "rewards/chosen": -0.13492245423166374, | |
| "rewards/margins": 2.425167830849466, | |
| "rewards/rejected": -2.56009028508113, | |
| "step": 188 | |
| }, | |
| { | |
| "epoch": 0.3271311120726958, | |
| "grad_norm": 19.875, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -12387616.842105264, | |
| "logits/rejected": -8763428.923076924, | |
| "logps/chosen": -166.91708213404604, | |
| "logps/rejected": -238.5693359375, | |
| "loss": 0.3205, | |
| "rewards/chosen": 0.12465482009084601, | |
| "rewards/margins": 2.8608485503717955, | |
| "rewards/rejected": -2.7361937302809496, | |
| "step": 189 | |
| }, | |
| { | |
| "epoch": 0.32886196451752486, | |
| "grad_norm": 20.0, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -11473281.454545455, | |
| "logits/rejected": -16561514.666666666, | |
| "logps/chosen": -248.29432262073863, | |
| "logps/rejected": -292.7100074404762, | |
| "loss": 0.2314, | |
| "rewards/chosen": 0.17398832061073996, | |
| "rewards/margins": 3.0009191387143486, | |
| "rewards/rejected": -2.826930818103609, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 0.330592816962354, | |
| "grad_norm": 16.25, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -8514306.823529411, | |
| "logits/rejected": -15071434.666666666, | |
| "logps/chosen": -131.8534725413603, | |
| "logps/rejected": -270.87555338541665, | |
| "loss": 0.3024, | |
| "rewards/chosen": 0.18799910825841568, | |
| "rewards/margins": 3.0057781406477386, | |
| "rewards/rejected": -2.817779032389323, | |
| "step": 191 | |
| }, | |
| { | |
| "epoch": 0.33232366940718305, | |
| "grad_norm": 19.25, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -1210433.894736842, | |
| "logits/rejected": -8759064.0, | |
| "logps/chosen": -148.5827765213816, | |
| "logps/rejected": -183.75860126201923, | |
| "loss": 0.3729, | |
| "rewards/chosen": -0.010686732436481276, | |
| "rewards/margins": 2.6109067429053154, | |
| "rewards/rejected": -2.621593475341797, | |
| "step": 192 | |
| }, | |
| { | |
| "epoch": 0.3340545218520121, | |
| "grad_norm": 18.5, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 1876100.6153846155, | |
| "logits/rejected": -12178267.789473685, | |
| "logps/chosen": -158.33451021634616, | |
| "logps/rejected": -272.60916940789474, | |
| "loss": 0.2534, | |
| "rewards/chosen": 0.049141957209660456, | |
| "rewards/margins": 3.0719871752657872, | |
| "rewards/rejected": -3.0228452180561267, | |
| "step": 193 | |
| }, | |
| { | |
| "epoch": 0.3357853742968412, | |
| "grad_norm": 18.875, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -9391846.0, | |
| "logits/rejected": -7076478.5, | |
| "logps/chosen": -197.68260192871094, | |
| "logps/rejected": -201.0494384765625, | |
| "loss": 0.2827, | |
| "rewards/chosen": 0.280254989862442, | |
| "rewards/margins": 3.4145003855228424, | |
| "rewards/rejected": -3.1342453956604004, | |
| "step": 194 | |
| }, | |
| { | |
| "epoch": 0.33751622674167026, | |
| "grad_norm": 18.875, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -19537841.066666666, | |
| "logits/rejected": -4787949.176470588, | |
| "logps/chosen": -203.43776041666666, | |
| "logps/rejected": -216.7041015625, | |
| "loss": 0.3368, | |
| "rewards/chosen": 0.03912758032480876, | |
| "rewards/margins": 2.348604769332736, | |
| "rewards/rejected": -2.3094771890079273, | |
| "step": 195 | |
| }, | |
| { | |
| "epoch": 0.3392470791864994, | |
| "grad_norm": 16.625, | |
| "kl": 0.16342926025390625, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -3170068.8421052634, | |
| "logits/rejected": -18859544.615384616, | |
| "logps/chosen": -124.50954718338816, | |
| "logps/rejected": -328.8591120793269, | |
| "loss": 0.2909, | |
| "rewards/chosen": 0.2587398478859349, | |
| "rewards/margins": 3.9374563607127078, | |
| "rewards/rejected": -3.6787165128267727, | |
| "step": 196 | |
| }, | |
| { | |
| "epoch": 0.34097793163132845, | |
| "grad_norm": 21.375, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 454123.25, | |
| "logits/rejected": -4452693.0, | |
| "logps/chosen": -233.21722412109375, | |
| "logps/rejected": -264.53143310546875, | |
| "loss": 0.2989, | |
| "rewards/chosen": -0.011935576796531677, | |
| "rewards/margins": 3.860817089676857, | |
| "rewards/rejected": -3.8727526664733887, | |
| "step": 197 | |
| }, | |
| { | |
| "epoch": 0.3427087840761575, | |
| "grad_norm": 16.375, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -3420107.466666667, | |
| "logits/rejected": -14321411.764705881, | |
| "logps/chosen": -135.79720052083334, | |
| "logps/rejected": -363.07223690257354, | |
| "loss": 0.2492, | |
| "rewards/chosen": 0.19408594767252604, | |
| "rewards/margins": 3.735214442832797, | |
| "rewards/rejected": -3.541128495160271, | |
| "step": 198 | |
| }, | |
| { | |
| "epoch": 0.3444396365209866, | |
| "grad_norm": 21.75, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -5829539.6, | |
| "logits/rejected": -15839922.666666666, | |
| "logps/chosen": -191.86925048828124, | |
| "logps/rejected": -276.8868408203125, | |
| "loss": 0.3512, | |
| "rewards/chosen": 0.14528814554214478, | |
| "rewards/margins": 2.9095884919166566, | |
| "rewards/rejected": -2.7643003463745117, | |
| "step": 199 | |
| }, | |
| { | |
| "epoch": 0.34617048896581565, | |
| "grad_norm": 17.125, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -3732642.0, | |
| "logits/rejected": -6055160.0, | |
| "logps/chosen": -145.93505859375, | |
| "logps/rejected": -213.4742889404297, | |
| "loss": 0.2826, | |
| "rewards/chosen": 0.03191981464624405, | |
| "rewards/margins": 3.5864234939217567, | |
| "rewards/rejected": -3.5545036792755127, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.3479013414106447, | |
| "grad_norm": 17.5, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -10997382.666666666, | |
| "logits/rejected": -13361217.6, | |
| "logps/chosen": -214.18802897135416, | |
| "logps/rejected": -290.353857421875, | |
| "loss": 0.2108, | |
| "rewards/chosen": 0.08698128660519917, | |
| "rewards/margins": 3.8983208556969964, | |
| "rewards/rejected": -3.811339569091797, | |
| "step": 201 | |
| }, | |
| { | |
| "epoch": 0.34963219385547384, | |
| "grad_norm": 15.875, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -16647024.0, | |
| "logits/rejected": -16219051.789473685, | |
| "logps/chosen": -266.0360576923077, | |
| "logps/rejected": -278.2851305509868, | |
| "loss": 0.2141, | |
| "rewards/chosen": 0.3400090290949895, | |
| "rewards/margins": 4.1297148075180985, | |
| "rewards/rejected": -3.7897057784231087, | |
| "step": 202 | |
| }, | |
| { | |
| "epoch": 0.3513630463003029, | |
| "grad_norm": 20.625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -14588352.0, | |
| "logits/rejected": -4774448.842105263, | |
| "logps/chosen": -257.1218825120192, | |
| "logps/rejected": -202.72636975740133, | |
| "loss": 0.2567, | |
| "rewards/chosen": -0.031910451558920055, | |
| "rewards/margins": 3.413597200322248, | |
| "rewards/rejected": -3.445507651881168, | |
| "step": 203 | |
| }, | |
| { | |
| "epoch": 0.353093898745132, | |
| "grad_norm": 21.625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -3514067.5, | |
| "logits/rejected": -21243462.0, | |
| "logps/chosen": -222.9899139404297, | |
| "logps/rejected": -302.18115234375, | |
| "loss": 0.2852, | |
| "rewards/chosen": -0.013363361358642578, | |
| "rewards/margins": 3.822953462600708, | |
| "rewards/rejected": -3.8363168239593506, | |
| "step": 204 | |
| }, | |
| { | |
| "epoch": 0.35482475118996104, | |
| "grad_norm": 23.375, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 10442356.444444444, | |
| "logits/rejected": -5046877.142857143, | |
| "logps/chosen": -239.98092990451389, | |
| "logps/rejected": -214.3353271484375, | |
| "loss": 0.3217, | |
| "rewards/chosen": 0.006140223807758755, | |
| "rewards/margins": 3.9656730977788803, | |
| "rewards/rejected": -3.9595328739711215, | |
| "step": 205 | |
| }, | |
| { | |
| "epoch": 0.3565556036347901, | |
| "grad_norm": 17.125, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -3316098.5714285714, | |
| "logits/rejected": -3345895.5555555555, | |
| "logps/chosen": -141.17440359933036, | |
| "logps/rejected": -268.8661838107639, | |
| "loss": 0.2669, | |
| "rewards/chosen": -0.06388027327401298, | |
| "rewards/margins": 3.7650589526645724, | |
| "rewards/rejected": -3.8289392259385853, | |
| "step": 206 | |
| }, | |
| { | |
| "epoch": 0.35828645607961923, | |
| "grad_norm": 24.125, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -10579598.476190476, | |
| "logits/rejected": -7490005.818181818, | |
| "logps/chosen": -215.40415736607142, | |
| "logps/rejected": -398.5939275568182, | |
| "loss": 0.3157, | |
| "rewards/chosen": 0.11144000007992699, | |
| "rewards/margins": 5.1400641011985355, | |
| "rewards/rejected": -5.028624101118608, | |
| "step": 207 | |
| }, | |
| { | |
| "epoch": 0.3600173085244483, | |
| "grad_norm": 18.75, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -2552588.470588235, | |
| "logits/rejected": 11076235.733333332, | |
| "logps/chosen": -175.71145450367646, | |
| "logps/rejected": -340.4918619791667, | |
| "loss": 0.293, | |
| "rewards/chosen": 0.09705781235414393, | |
| "rewards/margins": 3.936987011572894, | |
| "rewards/rejected": -3.83992919921875, | |
| "step": 208 | |
| }, | |
| { | |
| "epoch": 0.36174816096927737, | |
| "grad_norm": 17.75, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -8181181.0, | |
| "logits/rejected": -4103885.5, | |
| "logps/chosen": -175.54034423828125, | |
| "logps/rejected": -195.105712890625, | |
| "loss": 0.3223, | |
| "rewards/chosen": 0.006002817302942276, | |
| "rewards/margins": 2.481421146541834, | |
| "rewards/rejected": -2.4754183292388916, | |
| "step": 209 | |
| }, | |
| { | |
| "epoch": 0.36347901341410643, | |
| "grad_norm": 21.625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -6023152.0, | |
| "logits/rejected": -6930216.615384615, | |
| "logps/chosen": -220.12386924342104, | |
| "logps/rejected": -259.09130859375, | |
| "loss": 0.3206, | |
| "rewards/chosen": 0.02514595577591344, | |
| "rewards/margins": 3.885440004378678, | |
| "rewards/rejected": -3.8602940486027646, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 0.3652098658589355, | |
| "grad_norm": 17.375, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -8873345.846153846, | |
| "logits/rejected": -4841296.421052632, | |
| "logps/chosen": -222.28448016826923, | |
| "logps/rejected": -257.08958675986844, | |
| "loss": 0.2588, | |
| "rewards/chosen": -0.17216739287743202, | |
| "rewards/margins": 3.402585816286836, | |
| "rewards/rejected": -3.574753209164268, | |
| "step": 211 | |
| }, | |
| { | |
| "epoch": 0.3669407183037646, | |
| "grad_norm": 18.5, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 1087914.2666666666, | |
| "logits/rejected": -13504388.705882354, | |
| "logps/chosen": -207.50533854166667, | |
| "logps/rejected": -327.5465303308824, | |
| "loss": 0.2455, | |
| "rewards/chosen": 0.09607280890146891, | |
| "rewards/margins": 4.1472624559028475, | |
| "rewards/rejected": -4.051189647001379, | |
| "step": 212 | |
| }, | |
| { | |
| "epoch": 0.3686715707485937, | |
| "grad_norm": 18.375, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -12526222.933333334, | |
| "logits/rejected": -2303273.882352941, | |
| "logps/chosen": -217.412255859375, | |
| "logps/rejected": -159.63197954963235, | |
| "loss": 0.2619, | |
| "rewards/chosen": 0.2408916155497233, | |
| "rewards/margins": 3.14007298151652, | |
| "rewards/rejected": -2.899181365966797, | |
| "step": 213 | |
| }, | |
| { | |
| "epoch": 0.37040242319342276, | |
| "grad_norm": 21.375, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -6768866.105263158, | |
| "logits/rejected": -11241085.538461538, | |
| "logps/chosen": -209.8044305098684, | |
| "logps/rejected": -294.76034780649036, | |
| "loss": 0.3237, | |
| "rewards/chosen": 0.22350662632992394, | |
| "rewards/margins": 2.894371611869287, | |
| "rewards/rejected": -2.670864985539363, | |
| "step": 214 | |
| }, | |
| { | |
| "epoch": 0.3721332756382518, | |
| "grad_norm": 20.0, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -15369493.333333334, | |
| "logits/rejected": 2045517.2857142857, | |
| "logps/chosen": -224.15772840711804, | |
| "logps/rejected": -162.84868512834822, | |
| "loss": 0.2959, | |
| "rewards/chosen": 0.2034378316667345, | |
| "rewards/margins": 3.11308999667092, | |
| "rewards/rejected": -2.9096521650041853, | |
| "step": 215 | |
| }, | |
| { | |
| "epoch": 0.3738641280830809, | |
| "grad_norm": 20.75, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -20242884.0, | |
| "logits/rejected": -7449432.0, | |
| "logps/chosen": -279.4679260253906, | |
| "logps/rejected": -204.05104064941406, | |
| "loss": 0.2813, | |
| "rewards/chosen": 0.30105486512184143, | |
| "rewards/margins": 3.196340948343277, | |
| "rewards/rejected": -2.8952860832214355, | |
| "step": 216 | |
| }, | |
| { | |
| "epoch": 0.37559498052791, | |
| "grad_norm": 17.5, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -5433502.909090909, | |
| "logits/rejected": 500345.14285714284, | |
| "logps/chosen": -189.3955078125, | |
| "logps/rejected": -303.5468982514881, | |
| "loss": 0.2097, | |
| "rewards/chosen": 0.4285439144481312, | |
| "rewards/margins": 3.622101653705944, | |
| "rewards/rejected": -3.1935577392578125, | |
| "step": 217 | |
| }, | |
| { | |
| "epoch": 0.3773258329727391, | |
| "grad_norm": 18.75, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -8591011.333333334, | |
| "logits/rejected": -6802850.4, | |
| "logps/chosen": -284.0384928385417, | |
| "logps/rejected": -196.76927490234374, | |
| "loss": 0.2485, | |
| "rewards/chosen": 0.3856252034505208, | |
| "rewards/margins": 3.309361775716146, | |
| "rewards/rejected": -2.923736572265625, | |
| "step": 218 | |
| }, | |
| { | |
| "epoch": 0.37905668541756815, | |
| "grad_norm": 14.0, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 4748231.2, | |
| "logits/rejected": -13977383.272727273, | |
| "logps/chosen": -120.3367431640625, | |
| "logps/rejected": -259.1859685724432, | |
| "loss": 0.2385, | |
| "rewards/chosen": -0.07536518573760986, | |
| "rewards/margins": 3.3064329515803945, | |
| "rewards/rejected": -3.3817981373180044, | |
| "step": 219 | |
| }, | |
| { | |
| "epoch": 0.3807875378623972, | |
| "grad_norm": 18.625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -2363141.230769231, | |
| "logits/rejected": -15810723.368421054, | |
| "logps/chosen": -170.66443810096155, | |
| "logps/rejected": -376.7901675575658, | |
| "loss": 0.2529, | |
| "rewards/chosen": 0.12850810931279108, | |
| "rewards/margins": 3.2996522936261132, | |
| "rewards/rejected": -3.1711441843133223, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 0.3825183903072263, | |
| "grad_norm": 19.25, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -11173596.8, | |
| "logits/rejected": -14623304.470588235, | |
| "logps/chosen": -228.55849609375, | |
| "logps/rejected": -248.14662798713235, | |
| "loss": 0.2426, | |
| "rewards/chosen": 0.3422792116800944, | |
| "rewards/margins": 3.8824749011619417, | |
| "rewards/rejected": -3.5401956894818474, | |
| "step": 221 | |
| }, | |
| { | |
| "epoch": 0.3842492427520554, | |
| "grad_norm": 17.125, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -20544094.769230768, | |
| "logits/rejected": -15617290.105263159, | |
| "logps/chosen": -238.24759615384616, | |
| "logps/rejected": -270.76454564144734, | |
| "loss": 0.2101, | |
| "rewards/chosen": 0.37209848257211536, | |
| "rewards/margins": 4.252398904035931, | |
| "rewards/rejected": -3.880300421463816, | |
| "step": 222 | |
| }, | |
| { | |
| "epoch": 0.3859800951968845, | |
| "grad_norm": 18.875, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 2339170.5, | |
| "logits/rejected": -14668691.0, | |
| "logps/chosen": -168.8272247314453, | |
| "logps/rejected": -316.6446228027344, | |
| "loss": 0.2904, | |
| "rewards/chosen": 0.05972611904144287, | |
| "rewards/margins": 3.1063586473464966, | |
| "rewards/rejected": -3.0466325283050537, | |
| "step": 223 | |
| }, | |
| { | |
| "epoch": 0.38771094764171354, | |
| "grad_norm": 21.25, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -10121347.42857143, | |
| "logits/rejected": -9601993.777777778, | |
| "logps/chosen": -243.92325265066964, | |
| "logps/rejected": -305.10259331597223, | |
| "loss": 0.2482, | |
| "rewards/chosen": 0.2614833116531372, | |
| "rewards/margins": 3.4309277137120566, | |
| "rewards/rejected": -3.1694444020589194, | |
| "step": 224 | |
| }, | |
| { | |
| "epoch": 0.3894418000865426, | |
| "grad_norm": 18.625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 8372479.157894737, | |
| "logits/rejected": -18436361.846153848, | |
| "logps/chosen": -128.38247841282896, | |
| "logps/rejected": -282.94106820913464, | |
| "loss": 0.353, | |
| "rewards/chosen": -0.023535085351843583, | |
| "rewards/margins": 3.7766199674200913, | |
| "rewards/rejected": -3.800155052771935, | |
| "step": 225 | |
| }, | |
| { | |
| "epoch": 0.3911726525313717, | |
| "grad_norm": 21.875, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -1354891.6842105263, | |
| "logits/rejected": -14058054.153846154, | |
| "logps/chosen": -161.87158203125, | |
| "logps/rejected": -339.60730919471155, | |
| "loss": 0.304, | |
| "rewards/chosen": 0.12897560470982602, | |
| "rewards/margins": 4.465609140241677, | |
| "rewards/rejected": -4.336633535531851, | |
| "step": 226 | |
| }, | |
| { | |
| "epoch": 0.3929035049762008, | |
| "grad_norm": 21.375, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -10190957.176470589, | |
| "logits/rejected": -10005033.6, | |
| "logps/chosen": -191.16943359375, | |
| "logps/rejected": -214.92547200520832, | |
| "loss": 0.3543, | |
| "rewards/chosen": -0.27459220325245576, | |
| "rewards/margins": 2.7234729822944193, | |
| "rewards/rejected": -2.998065185546875, | |
| "step": 227 | |
| }, | |
| { | |
| "epoch": 0.39463435742102987, | |
| "grad_norm": 16.75, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -12754082.133333333, | |
| "logits/rejected": 9428894.11764706, | |
| "logps/chosen": -160.12646484375, | |
| "logps/rejected": -193.08636833639707, | |
| "loss": 0.2904, | |
| "rewards/chosen": 0.34527934392293297, | |
| "rewards/margins": 2.9829243921766095, | |
| "rewards/rejected": -2.6376450482536766, | |
| "step": 228 | |
| }, | |
| { | |
| "epoch": 0.39636520986585894, | |
| "grad_norm": 19.625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -11040588.631578946, | |
| "logits/rejected": -28768635.076923076, | |
| "logps/chosen": -202.25439453125, | |
| "logps/rejected": -298.80213341346155, | |
| "loss": 0.2818, | |
| "rewards/chosen": 0.2841626719424599, | |
| "rewards/margins": 4.040860400026144, | |
| "rewards/rejected": -3.756697728083684, | |
| "step": 229 | |
| }, | |
| { | |
| "epoch": 0.398096062310688, | |
| "grad_norm": 18.625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -16696243.2, | |
| "logits/rejected": 782789.7058823529, | |
| "logps/chosen": -209.39425455729167, | |
| "logps/rejected": -243.5696518841912, | |
| "loss": 0.2405, | |
| "rewards/chosen": 0.21385353406270344, | |
| "rewards/margins": 3.5233376792832916, | |
| "rewards/rejected": -3.3094841452205883, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 0.39982691475551707, | |
| "grad_norm": 17.125, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -6847610.857142857, | |
| "logits/rejected": -341155.47222222225, | |
| "logps/chosen": -145.1298566545759, | |
| "logps/rejected": -159.03776041666666, | |
| "loss": 0.3048, | |
| "rewards/chosen": 0.034968899829047065, | |
| "rewards/margins": 2.7618577683728835, | |
| "rewards/rejected": -2.7268888685438366, | |
| "step": 231 | |
| }, | |
| { | |
| "epoch": 0.4015577672003462, | |
| "grad_norm": 19.0, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -14881117.0, | |
| "logits/rejected": -15492494.0, | |
| "logps/chosen": -207.72711181640625, | |
| "logps/rejected": -320.58319091796875, | |
| "loss": 0.2664, | |
| "rewards/chosen": 0.16509190201759338, | |
| "rewards/margins": 4.239029794931412, | |
| "rewards/rejected": -4.073937892913818, | |
| "step": 232 | |
| }, | |
| { | |
| "epoch": 0.40328861964517526, | |
| "grad_norm": 20.25, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -5929799.529411765, | |
| "logits/rejected": -12552819.2, | |
| "logps/chosen": -215.2191664751838, | |
| "logps/rejected": -290.16845703125, | |
| "loss": 0.2607, | |
| "rewards/chosen": 0.19065551196827607, | |
| "rewards/margins": 4.717087295008641, | |
| "rewards/rejected": -4.526431783040365, | |
| "step": 233 | |
| }, | |
| { | |
| "epoch": 0.40501947209000433, | |
| "grad_norm": 17.0, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 8185308.8, | |
| "logits/rejected": -14831882.352941176, | |
| "logps/chosen": -165.01144205729167, | |
| "logps/rejected": -273.51809512867646, | |
| "loss": 0.2833, | |
| "rewards/chosen": -0.0994392474492391, | |
| "rewards/margins": 3.8827245745004393, | |
| "rewards/rejected": -3.9821638219496784, | |
| "step": 234 | |
| }, | |
| { | |
| "epoch": 0.4067503245348334, | |
| "grad_norm": 21.75, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -12612798.315789474, | |
| "logits/rejected": -19476348.307692308, | |
| "logps/chosen": -231.91385690789474, | |
| "logps/rejected": -316.7388446514423, | |
| "loss": 0.3007, | |
| "rewards/chosen": 0.1857273955094187, | |
| "rewards/margins": 5.110347803787664, | |
| "rewards/rejected": -4.924620408278245, | |
| "step": 235 | |
| }, | |
| { | |
| "epoch": 0.40848117697966246, | |
| "grad_norm": 14.1875, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -13798296.0, | |
| "logits/rejected": -13150624.0, | |
| "logps/chosen": -184.28253173828125, | |
| "logps/rejected": -293.38729580965907, | |
| "loss": 0.2181, | |
| "rewards/chosen": -0.08902863264083863, | |
| "rewards/margins": 3.767916405200958, | |
| "rewards/rejected": -3.856945037841797, | |
| "step": 236 | |
| }, | |
| { | |
| "epoch": 0.4102120294244916, | |
| "grad_norm": 19.625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -5185518.933333334, | |
| "logits/rejected": -10506853.647058824, | |
| "logps/chosen": -195.71438802083333, | |
| "logps/rejected": -266.6700080422794, | |
| "loss": 0.2492, | |
| "rewards/chosen": 0.16841630935668944, | |
| "rewards/margins": 4.66889471727259, | |
| "rewards/rejected": -4.5004784079159, | |
| "step": 237 | |
| }, | |
| { | |
| "epoch": 0.41194288186932065, | |
| "grad_norm": 17.75, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -120490.13333333333, | |
| "logits/rejected": -4157334.588235294, | |
| "logps/chosen": -165.81774088541667, | |
| "logps/rejected": -338.9480985753676, | |
| "loss": 0.2444, | |
| "rewards/chosen": 0.16940480868021648, | |
| "rewards/margins": 3.9526734567156026, | |
| "rewards/rejected": -3.783268648035386, | |
| "step": 238 | |
| }, | |
| { | |
| "epoch": 0.4136737343141497, | |
| "grad_norm": 21.75, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -13063847.466666667, | |
| "logits/rejected": -7603120.0, | |
| "logps/chosen": -217.052294921875, | |
| "logps/rejected": -281.8909696691176, | |
| "loss": 0.2602, | |
| "rewards/chosen": 0.0732549508412679, | |
| "rewards/margins": 3.9795507272084554, | |
| "rewards/rejected": -3.9062957763671875, | |
| "step": 239 | |
| }, | |
| { | |
| "epoch": 0.4154045867589788, | |
| "grad_norm": 19.25, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -13696624.0, | |
| "logits/rejected": 21262386.82352941, | |
| "logps/chosen": -188.62125651041666, | |
| "logps/rejected": -326.68488625919116, | |
| "loss": 0.2636, | |
| "rewards/chosen": 0.14170858065287273, | |
| "rewards/margins": 3.8223358425439575, | |
| "rewards/rejected": -3.6806272618910847, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 0.41713543920380786, | |
| "grad_norm": 15.5, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -11527736.615384616, | |
| "logits/rejected": -13245899.789473685, | |
| "logps/chosen": -171.26971905048077, | |
| "logps/rejected": -316.2771638569079, | |
| "loss": 0.2359, | |
| "rewards/chosen": 0.039726394873399004, | |
| "rewards/margins": 4.057520953749838, | |
| "rewards/rejected": -4.017794558876439, | |
| "step": 241 | |
| }, | |
| { | |
| "epoch": 0.418866291648637, | |
| "grad_norm": 20.125, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -10542560.0, | |
| "logits/rejected": -15128954.0, | |
| "logps/chosen": -197.97857666015625, | |
| "logps/rejected": -249.15786743164062, | |
| "loss": 0.2893, | |
| "rewards/chosen": -0.0119645856320858, | |
| "rewards/margins": 3.6342065073549747, | |
| "rewards/rejected": -3.6461710929870605, | |
| "step": 242 | |
| }, | |
| { | |
| "epoch": 0.42059714409346605, | |
| "grad_norm": 22.5, | |
| "kl": 0.13220763206481934, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -17448896.0, | |
| "logits/rejected": -10536552.533333333, | |
| "logps/chosen": -242.55230353860293, | |
| "logps/rejected": -180.8974609375, | |
| "loss": 0.3269, | |
| "rewards/chosen": 0.07533069217906278, | |
| "rewards/margins": 3.0032341854245055, | |
| "rewards/rejected": -2.927903493245443, | |
| "step": 243 | |
| }, | |
| { | |
| "epoch": 0.4223279965382951, | |
| "grad_norm": 16.875, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -15767290.181818182, | |
| "logits/rejected": -11785589.333333334, | |
| "logps/chosen": -199.07419655539772, | |
| "logps/rejected": -267.2129371279762, | |
| "loss": 0.1908, | |
| "rewards/chosen": 0.3069478381763805, | |
| "rewards/margins": 3.4843519693845275, | |
| "rewards/rejected": -3.177404131208147, | |
| "step": 244 | |
| }, | |
| { | |
| "epoch": 0.4240588489831242, | |
| "grad_norm": 16.125, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -9023009.777777778, | |
| "logits/rejected": -8307068.0, | |
| "logps/chosen": -138.5511474609375, | |
| "logps/rejected": -191.49358258928572, | |
| "loss": 0.2808, | |
| "rewards/chosen": 0.27963558832804364, | |
| "rewards/margins": 3.714873745327904, | |
| "rewards/rejected": -3.4352381569998607, | |
| "step": 245 | |
| }, | |
| { | |
| "epoch": 0.42578970142795325, | |
| "grad_norm": 14.8125, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -12051193.6, | |
| "logits/rejected": -10616396.235294119, | |
| "logps/chosen": -171.37771809895833, | |
| "logps/rejected": -222.20760569852942, | |
| "loss": 0.2502, | |
| "rewards/chosen": 0.1699681282043457, | |
| "rewards/margins": 4.40653590595021, | |
| "rewards/rejected": -4.236567777745864, | |
| "step": 246 | |
| }, | |
| { | |
| "epoch": 0.42752055387278237, | |
| "grad_norm": 19.5, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -12088685.714285715, | |
| "logits/rejected": -5290852.0, | |
| "logps/chosen": -208.72258649553572, | |
| "logps/rejected": -252.75640190972223, | |
| "loss": 0.2659, | |
| "rewards/chosen": 0.08243453502655029, | |
| "rewards/margins": 3.2210644483566284, | |
| "rewards/rejected": -3.138629913330078, | |
| "step": 247 | |
| }, | |
| { | |
| "epoch": 0.42925140631761144, | |
| "grad_norm": 16.625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -18058203.2, | |
| "logits/rejected": -10416639.272727273, | |
| "logps/chosen": -224.68359375, | |
| "logps/rejected": -289.28861860795456, | |
| "loss": 0.2142, | |
| "rewards/chosen": -0.14810900688171386, | |
| "rewards/margins": 3.584905880147761, | |
| "rewards/rejected": -3.7330148870294746, | |
| "step": 248 | |
| }, | |
| { | |
| "epoch": 0.4309822587624405, | |
| "grad_norm": 18.375, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -15116648.533333333, | |
| "logits/rejected": -3318948.0, | |
| "logps/chosen": -177.97216796875, | |
| "logps/rejected": -264.22144990808823, | |
| "loss": 0.2523, | |
| "rewards/chosen": 0.038343381881713864, | |
| "rewards/margins": 4.054038841584149, | |
| "rewards/rejected": -4.015695459702435, | |
| "step": 249 | |
| }, | |
| { | |
| "epoch": 0.4327131112072696, | |
| "grad_norm": 22.75, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -11157197.47368421, | |
| "logits/rejected": 2493580.3076923075, | |
| "logps/chosen": -177.07334498355263, | |
| "logps/rejected": -290.3555438701923, | |
| "loss": 0.3557, | |
| "rewards/chosen": -0.19027650983710037, | |
| "rewards/margins": 3.0934826806489273, | |
| "rewards/rejected": -3.2837591904860277, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 0.43444396365209864, | |
| "grad_norm": 20.375, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -16668712.533333333, | |
| "logits/rejected": -3976555.294117647, | |
| "logps/chosen": -244.20847981770834, | |
| "logps/rejected": -327.9715935202206, | |
| "loss": 0.2648, | |
| "rewards/chosen": 0.03703808784484863, | |
| "rewards/margins": 4.71754776730257, | |
| "rewards/rejected": -4.680509679457721, | |
| "step": 251 | |
| }, | |
| { | |
| "epoch": 0.43617481609692776, | |
| "grad_norm": 17.625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 5272862.545454546, | |
| "logits/rejected": -6936409.904761905, | |
| "logps/chosen": -229.92167524857953, | |
| "logps/rejected": -213.8749534970238, | |
| "loss": 0.2408, | |
| "rewards/chosen": -0.30214368213306775, | |
| "rewards/margins": 3.3398687777581153, | |
| "rewards/rejected": -3.642012459891183, | |
| "step": 252 | |
| }, | |
| { | |
| "epoch": 0.43790566854175683, | |
| "grad_norm": 19.625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -3558934.933333333, | |
| "logits/rejected": -2213124.2352941176, | |
| "logps/chosen": -207.47721354166666, | |
| "logps/rejected": -196.65370806525735, | |
| "loss": 0.2565, | |
| "rewards/chosen": 0.18845229148864745, | |
| "rewards/margins": 3.44047677376691, | |
| "rewards/rejected": -3.2520244822782627, | |
| "step": 253 | |
| }, | |
| { | |
| "epoch": 0.4396365209865859, | |
| "grad_norm": 20.25, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -3256860.5714285714, | |
| "logits/rejected": -1937111.3333333333, | |
| "logps/chosen": -222.31358119419642, | |
| "logps/rejected": -198.45515950520834, | |
| "loss": 0.2842, | |
| "rewards/chosen": -0.027769644345555986, | |
| "rewards/margins": 3.096243832556028, | |
| "rewards/rejected": -3.124013476901584, | |
| "step": 254 | |
| }, | |
| { | |
| "epoch": 0.44136737343141497, | |
| "grad_norm": 20.375, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 2231512.0, | |
| "logits/rejected": -15707685.714285715, | |
| "logps/chosen": -224.28355577256946, | |
| "logps/rejected": -377.65659877232144, | |
| "loss": 0.283, | |
| "rewards/chosen": 0.1279101769129435, | |
| "rewards/margins": 4.255186188788642, | |
| "rewards/rejected": -4.127276011875698, | |
| "step": 255 | |
| }, | |
| { | |
| "epoch": 0.44309822587624403, | |
| "grad_norm": 19.125, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -3650349.0, | |
| "logits/rejected": -25763404.0, | |
| "logps/chosen": -175.8751220703125, | |
| "logps/rejected": -384.6330261230469, | |
| "loss": 0.2993, | |
| "rewards/chosen": -0.2350717931985855, | |
| "rewards/margins": 3.420280560851097, | |
| "rewards/rejected": -3.6553523540496826, | |
| "step": 256 | |
| }, | |
| { | |
| "epoch": 0.4448290783210731, | |
| "grad_norm": 22.375, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -8227142.095238095, | |
| "logits/rejected": -3991596.0, | |
| "logps/chosen": -204.65394810267858, | |
| "logps/rejected": -230.69422496448863, | |
| "loss": 0.3679, | |
| "rewards/chosen": -0.06990920929681688, | |
| "rewards/margins": 4.00661361785162, | |
| "rewards/rejected": -4.0765228271484375, | |
| "step": 257 | |
| }, | |
| { | |
| "epoch": 0.4465599307659022, | |
| "grad_norm": 20.625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -9966890.105263159, | |
| "logits/rejected": -14217319.384615384, | |
| "logps/chosen": -199.6491827713816, | |
| "logps/rejected": -275.9331242487981, | |
| "loss": 0.3237, | |
| "rewards/chosen": 0.01347437344099346, | |
| "rewards/margins": 4.150257386418007, | |
| "rewards/rejected": -4.136783012977014, | |
| "step": 258 | |
| }, | |
| { | |
| "epoch": 0.4482907832107313, | |
| "grad_norm": 20.0, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -13260032.94117647, | |
| "logits/rejected": -3421154.6666666665, | |
| "logps/chosen": -238.68637982536765, | |
| "logps/rejected": -267.1708658854167, | |
| "loss": 0.2502, | |
| "rewards/chosen": 0.2879646806155934, | |
| "rewards/margins": 4.020605773551791, | |
| "rewards/rejected": -3.732641092936198, | |
| "step": 259 | |
| }, | |
| { | |
| "epoch": 0.45002163565556036, | |
| "grad_norm": 18.375, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -20596262.85714286, | |
| "logits/rejected": -7963298.666666667, | |
| "logps/chosen": -236.05545479910714, | |
| "logps/rejected": -297.91436089409723, | |
| "loss": 0.2213, | |
| "rewards/chosen": 0.6324899537222726, | |
| "rewards/margins": 4.426475108615936, | |
| "rewards/rejected": -3.7939851548936634, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 0.4517524881003894, | |
| "grad_norm": 15.9375, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -14417637.333333334, | |
| "logits/rejected": -13684048.0, | |
| "logps/chosen": -196.896435546875, | |
| "logps/rejected": -312.3822380514706, | |
| "loss": 0.1969, | |
| "rewards/chosen": 0.5348507563273112, | |
| "rewards/margins": 4.734749868804333, | |
| "rewards/rejected": -4.199899112477022, | |
| "step": 261 | |
| }, | |
| { | |
| "epoch": 0.4534833405452185, | |
| "grad_norm": 19.625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -24702128.0, | |
| "logits/rejected": -9365918.222222222, | |
| "logps/chosen": -337.36460658482144, | |
| "logps/rejected": -256.26673719618054, | |
| "loss": 0.2306, | |
| "rewards/chosen": 0.24765947886875697, | |
| "rewards/margins": 4.048638669271318, | |
| "rewards/rejected": -3.8009791904025607, | |
| "step": 262 | |
| }, | |
| { | |
| "epoch": 0.4552141929900476, | |
| "grad_norm": 19.0, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -12827164.631578946, | |
| "logits/rejected": -11524402.461538462, | |
| "logps/chosen": -181.87505139802633, | |
| "logps/rejected": -344.0277569110577, | |
| "loss": 0.2989, | |
| "rewards/chosen": 0.03691702453713668, | |
| "rewards/margins": 4.332692511409883, | |
| "rewards/rejected": -4.2957754868727465, | |
| "step": 263 | |
| }, | |
| { | |
| "epoch": 0.4569450454348767, | |
| "grad_norm": 20.625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -18784899.555555556, | |
| "logits/rejected": -12124728.0, | |
| "logps/chosen": -233.41818576388889, | |
| "logps/rejected": -315.662109375, | |
| "loss": 0.3184, | |
| "rewards/chosen": 0.08528125286102295, | |
| "rewards/margins": 3.363411920411246, | |
| "rewards/rejected": -3.278130667550223, | |
| "step": 264 | |
| }, | |
| { | |
| "epoch": 0.45867589787970575, | |
| "grad_norm": 19.125, | |
| "kl": 0.11611628532409668, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 1691305.142857143, | |
| "logits/rejected": -6767852.0, | |
| "logps/chosen": -285.6856166294643, | |
| "logps/rejected": -247.17659505208334, | |
| "loss": 0.2455, | |
| "rewards/chosen": 0.24254277774265834, | |
| "rewards/margins": 3.9829009184761657, | |
| "rewards/rejected": -3.740358140733507, | |
| "step": 265 | |
| }, | |
| { | |
| "epoch": 0.4604067503245348, | |
| "grad_norm": 20.5, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -11006114.4, | |
| "logits/rejected": -19388294.666666668, | |
| "logps/chosen": -188.975390625, | |
| "logps/rejected": -325.20001220703125, | |
| "loss": 0.3136, | |
| "rewards/chosen": 0.19371647834777833, | |
| "rewards/margins": 3.5510452111562096, | |
| "rewards/rejected": -3.357328732808431, | |
| "step": 266 | |
| }, | |
| { | |
| "epoch": 0.4621376027693639, | |
| "grad_norm": 16.75, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -17104797.53846154, | |
| "logits/rejected": -6318394.105263158, | |
| "logps/chosen": -190.21593299278845, | |
| "logps/rejected": -221.87461451480263, | |
| "loss": 0.2442, | |
| "rewards/chosen": 0.2514270819150485, | |
| "rewards/margins": 3.471239374716755, | |
| "rewards/rejected": -3.2198122928017066, | |
| "step": 267 | |
| }, | |
| { | |
| "epoch": 0.463868455214193, | |
| "grad_norm": 20.25, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -4255990.933333334, | |
| "logits/rejected": -16446816.0, | |
| "logps/chosen": -253.98564453125, | |
| "logps/rejected": -270.46570542279414, | |
| "loss": 0.2827, | |
| "rewards/chosen": -0.036499599615732826, | |
| "rewards/margins": 3.59961351109486, | |
| "rewards/rejected": -3.6361131107105926, | |
| "step": 268 | |
| }, | |
| { | |
| "epoch": 0.4655993076590221, | |
| "grad_norm": 17.0, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -11160579.2, | |
| "logits/rejected": -10607611.294117646, | |
| "logps/chosen": -202.769384765625, | |
| "logps/rejected": -240.4682186351103, | |
| "loss": 0.2514, | |
| "rewards/chosen": 0.4125640551249186, | |
| "rewards/margins": 3.6268443518993903, | |
| "rewards/rejected": -3.2142802967744717, | |
| "step": 269 | |
| }, | |
| { | |
| "epoch": 0.46733016010385114, | |
| "grad_norm": 18.625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -13615528.888888888, | |
| "logits/rejected": -6793550.857142857, | |
| "logps/chosen": -177.54604763454861, | |
| "logps/rejected": -297.06703404017856, | |
| "loss": 0.2885, | |
| "rewards/chosen": 0.25184231334262425, | |
| "rewards/margins": 3.5622371454087514, | |
| "rewards/rejected": -3.310394832066127, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 0.4690610125486802, | |
| "grad_norm": 21.0, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -2008512.4210526317, | |
| "logits/rejected": 1065751.2307692308, | |
| "logps/chosen": -230.33552631578948, | |
| "logps/rejected": -225.39548903245193, | |
| "loss": 0.2912, | |
| "rewards/chosen": 0.33769374144704717, | |
| "rewards/margins": 3.8641099331349977, | |
| "rewards/rejected": -3.5264161916879506, | |
| "step": 271 | |
| }, | |
| { | |
| "epoch": 0.4707918649935093, | |
| "grad_norm": 20.25, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -8079465.5, | |
| "logits/rejected": -4624071.0, | |
| "logps/chosen": -245.87030029296875, | |
| "logps/rejected": -286.6881103515625, | |
| "loss": 0.256, | |
| "rewards/chosen": 0.18378782272338867, | |
| "rewards/margins": 4.057176828384399, | |
| "rewards/rejected": -3.8733890056610107, | |
| "step": 272 | |
| }, | |
| { | |
| "epoch": 0.4725227174383384, | |
| "grad_norm": 17.25, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 34063.02272727273, | |
| "logits/rejected": 5305572.4, | |
| "logps/chosen": -97.9369229403409, | |
| "logps/rejected": -163.92113037109374, | |
| "loss": 0.362, | |
| "rewards/chosen": 0.07122220234437422, | |
| "rewards/margins": 3.202643482251601, | |
| "rewards/rejected": -3.1314212799072267, | |
| "step": 273 | |
| }, | |
| { | |
| "epoch": 0.47425356988316747, | |
| "grad_norm": 19.125, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -8214460.0, | |
| "logits/rejected": -21400504.0, | |
| "logps/chosen": -159.88619995117188, | |
| "logps/rejected": -304.96246337890625, | |
| "loss": 0.2742, | |
| "rewards/chosen": 0.257793664932251, | |
| "rewards/margins": 3.4434404373168945, | |
| "rewards/rejected": -3.1856467723846436, | |
| "step": 274 | |
| }, | |
| { | |
| "epoch": 0.47598442232799654, | |
| "grad_norm": 19.0, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -9168399.157894736, | |
| "logits/rejected": -14751639.384615384, | |
| "logps/chosen": -186.708251953125, | |
| "logps/rejected": -292.65147986778845, | |
| "loss": 0.2819, | |
| "rewards/chosen": 0.2397254642687346, | |
| "rewards/margins": 4.295501033304191, | |
| "rewards/rejected": -4.055775569035457, | |
| "step": 275 | |
| }, | |
| { | |
| "epoch": 0.4777152747728256, | |
| "grad_norm": 20.125, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -9120151.529411765, | |
| "logits/rejected": -2838455.7333333334, | |
| "logps/chosen": -179.58385512408088, | |
| "logps/rejected": -274.78151041666666, | |
| "loss": 0.2887, | |
| "rewards/chosen": 0.22131121859830968, | |
| "rewards/margins": 3.1119616499134137, | |
| "rewards/rejected": -2.890650431315104, | |
| "step": 276 | |
| }, | |
| { | |
| "epoch": 0.47944612721765467, | |
| "grad_norm": 17.625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -17451059.42857143, | |
| "logits/rejected": -19632037.333333332, | |
| "logps/chosen": -194.51698521205358, | |
| "logps/rejected": -261.6064453125, | |
| "loss": 0.255, | |
| "rewards/chosen": 0.24481826169150217, | |
| "rewards/margins": 3.8417675211316062, | |
| "rewards/rejected": -3.596949259440104, | |
| "step": 277 | |
| }, | |
| { | |
| "epoch": 0.4811769796624838, | |
| "grad_norm": 18.75, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -11978563.2, | |
| "logits/rejected": -11180960.94117647, | |
| "logps/chosen": -204.08976236979166, | |
| "logps/rejected": -332.37023207720586, | |
| "loss": 0.2457, | |
| "rewards/chosen": 0.2240306536356608, | |
| "rewards/margins": 3.5974735839694154, | |
| "rewards/rejected": -3.3734429303337548, | |
| "step": 278 | |
| }, | |
| { | |
| "epoch": 0.48290783210731286, | |
| "grad_norm": 17.5, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -4350533.866666666, | |
| "logits/rejected": -8590544.94117647, | |
| "logps/chosen": -151.40662434895833, | |
| "logps/rejected": -249.31729664522058, | |
| "loss": 0.2934, | |
| "rewards/chosen": 0.01796001394589742, | |
| "rewards/margins": 2.6930718967727585, | |
| "rewards/rejected": -2.675111882826861, | |
| "step": 279 | |
| }, | |
| { | |
| "epoch": 0.48463868455214193, | |
| "grad_norm": 19.375, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -10933750.4, | |
| "logits/rejected": -15255677.176470589, | |
| "logps/chosen": -171.53665364583333, | |
| "logps/rejected": -308.31043198529414, | |
| "loss": 0.2806, | |
| "rewards/chosen": 0.030906534194946288, | |
| "rewards/margins": 4.104628924762501, | |
| "rewards/rejected": -4.073722390567555, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 0.486369536996971, | |
| "grad_norm": 19.0, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -8110217.142857143, | |
| "logits/rejected": -18321370.181818184, | |
| "logps/chosen": -182.0908435639881, | |
| "logps/rejected": -304.40236594460225, | |
| "loss": 0.3329, | |
| "rewards/chosen": 0.09797722952706474, | |
| "rewards/margins": 3.671043247371525, | |
| "rewards/rejected": -3.5730660178444604, | |
| "step": 281 | |
| }, | |
| { | |
| "epoch": 0.48810038944180006, | |
| "grad_norm": 19.75, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -8963434.0, | |
| "logits/rejected": -6483309.0, | |
| "logps/chosen": -224.63430786132812, | |
| "logps/rejected": -291.1379699707031, | |
| "loss": 0.2706, | |
| "rewards/chosen": 0.08380473405122757, | |
| "rewards/margins": 3.3000806644558907, | |
| "rewards/rejected": -3.216275930404663, | |
| "step": 282 | |
| }, | |
| { | |
| "epoch": 0.4898312418866292, | |
| "grad_norm": 17.25, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -9069934.0, | |
| "logits/rejected": -22540276.0, | |
| "logps/chosen": -141.39923095703125, | |
| "logps/rejected": -317.710693359375, | |
| "loss": 0.3155, | |
| "rewards/chosen": -0.18840433657169342, | |
| "rewards/margins": 3.7216622680425644, | |
| "rewards/rejected": -3.910066604614258, | |
| "step": 283 | |
| }, | |
| { | |
| "epoch": 0.49156209433145825, | |
| "grad_norm": 23.5, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -11534782.315789474, | |
| "logits/rejected": -14227288.615384616, | |
| "logps/chosen": -347.19901315789474, | |
| "logps/rejected": -233.33997521033655, | |
| "loss": 0.2763, | |
| "rewards/chosen": 0.6239749506900185, | |
| "rewards/margins": 3.671866420792182, | |
| "rewards/rejected": -3.0478914701021633, | |
| "step": 284 | |
| }, | |
| { | |
| "epoch": 0.4932929467762873, | |
| "grad_norm": 19.0, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -7307938.52631579, | |
| "logits/rejected": -6305785.846153846, | |
| "logps/chosen": -182.99164782072367, | |
| "logps/rejected": -234.64518855168268, | |
| "loss": 0.3165, | |
| "rewards/chosen": 0.0814883081536544, | |
| "rewards/margins": 3.057790234986587, | |
| "rewards/rejected": -2.9763019268329325, | |
| "step": 285 | |
| }, | |
| { | |
| "epoch": 0.4950237992211164, | |
| "grad_norm": 17.125, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -2387972.2666666666, | |
| "logits/rejected": -10976164.705882354, | |
| "logps/chosen": -138.42179361979166, | |
| "logps/rejected": -257.4625459558824, | |
| "loss": 0.2974, | |
| "rewards/chosen": -0.04797365665435791, | |
| "rewards/margins": 3.496035608123331, | |
| "rewards/rejected": -3.5440092647776886, | |
| "step": 286 | |
| }, | |
| { | |
| "epoch": 0.49675465166594546, | |
| "grad_norm": 16.25, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -17446714.0, | |
| "logits/rejected": -4344534.5, | |
| "logps/chosen": -249.31387329101562, | |
| "logps/rejected": -275.65106201171875, | |
| "loss": 0.2517, | |
| "rewards/chosen": 0.2581062614917755, | |
| "rewards/margins": 3.798191577196121, | |
| "rewards/rejected": -3.5400853157043457, | |
| "step": 287 | |
| }, | |
| { | |
| "epoch": 0.4984855041107746, | |
| "grad_norm": 18.75, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -6731353.846153846, | |
| "logits/rejected": -10663467.789473685, | |
| "logps/chosen": -210.49735201322116, | |
| "logps/rejected": -270.64185855263156, | |
| "loss": 0.2199, | |
| "rewards/chosen": 0.23169115873483512, | |
| "rewards/margins": 4.13718504558208, | |
| "rewards/rejected": -3.905493886847245, | |
| "step": 288 | |
| }, | |
| { | |
| "epoch": 0.5002163565556036, | |
| "grad_norm": 20.5, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -12233035.2, | |
| "logits/rejected": -17931900.0, | |
| "logps/chosen": -199.65758056640624, | |
| "logps/rejected": -295.8255208333333, | |
| "loss": 0.3644, | |
| "rewards/chosen": -0.10184909105300903, | |
| "rewards/margins": 2.8798390905062354, | |
| "rewards/rejected": -2.9816881815592446, | |
| "step": 289 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 578, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 2, | |
| "save_steps": 289, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": false | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 2, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |