Model save

README.md

@@ -16,15 +16,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model was trained from scratch on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.4935
-- Rewards/chosen: -1.4744
-- Rewards/rejected: -2.5261
+- Loss: 0.4929
+- Rewards/chosen: -1.1416
+- Rewards/rejected: -2.1919
 - Rewards/accuracies: 0.7852
-- Rewards/margins: 1.0517
-- Logps/rejected: -508.4335
-- Logps/chosen: -401.7670
-- Logits/rejected: -0.6948
-- Logits/chosen: -0.8246
+- Rewards/margins: 1.0502
+- Logps/rejected: -469.4702
+- Logps/chosen: -363.3538
+- Logits/rejected: -1.6742
+- Logits/chosen: -1.7399
 
 ## Model description
 
@@ -61,7 +61,7 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.4747        | 0.9984 | 477  | 0.4935          | -1.4744        | -2.5261          | 0.7852             | 1.0517          | -508.4335      | -401.7670    | -0.6948         | -0.8246       |
+| 0.4838        | 0.9984 | 477  | 0.4929          | -1.1416        | -2.1919          | 0.7852             | 1.0502          | -469.4702      | -363.3538    | -1.6742         | -1.7399       |
 
 
 ### Framework versions

all_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.9984301412872841,
     "total_flos": 0.0,
-    "train_loss": 0.537537020957195,
-    "train_runtime": 15566.6343,
+    "train_loss": 0.5343590847351266,
+    "train_runtime": 13524.9429,
     "train_samples": 61134,
-    "train_samples_per_second": 3.927,
-    "train_steps_per_second": 0.031
+    "train_samples_per_second": 4.52,
+    "train_steps_per_second": 0.035
 }

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.9984301412872841,
     "total_flos": 0.0,
-    "train_loss": 0.537537020957195,
-    "train_runtime": 15566.6343,
+    "train_loss": 0.5343590847351266,
+    "train_runtime": 13524.9429,
     "train_samples": 61134,
-    "train_samples_per_second": 3.927,
-    "train_steps_per_second": 0.031
+    "train_samples_per_second": 4.52,
+    "train_steps_per_second": 0.035
 }

trainer_state.json

@@ -10,12 +10,12 @@
   "log_history": [
     {
       "epoch": 0.0020931449502878076,
-      "grad_norm": 9.420194644919581,
+      "grad_norm": 9.580024156968456,
       "learning_rate": 1.0416666666666666e-08,
-      "logits/chosen": -2.6372292041778564,
-      "logits/rejected": -2.557051181793213,
-      "logps/chosen": -303.9394836425781,
-      "logps/rejected": -351.8717041015625,
+      "logits/chosen": -2.430640935897827,
+      "logits/rejected": -2.325669050216675,
+      "logps/chosen": -301.1888122558594,
+      "logps/rejected": -346.8065185546875,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -25,733 +25,733 @@
     },
     {
       "epoch": 0.020931449502878074,
-      "grad_norm": 9.776596281600654,
+      "grad_norm": 9.535503163597756,
       "learning_rate": 1.0416666666666667e-07,
-      "logits/chosen": -2.366276741027832,
-      "logits/rejected": -2.316371202468872,
-      "logps/chosen": -292.4997253417969,
-      "logps/rejected": -264.6174621582031,
-      "loss": 0.6931,
-      "rewards/accuracies": 0.4583333432674408,
-      "rewards/chosen": 1.6382920875912532e-05,
-      "rewards/margins": 7.999181252671406e-05,
-      "rewards/rejected": -6.360887346090749e-05,
+      "logits/chosen": -2.121445655822754,
+      "logits/rejected": -2.055097818374634,
+      "logps/chosen": -287.1705322265625,
+      "logps/rejected": -259.10406494140625,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.4652777910232544,
+      "rewards/chosen": 0.00018805032595992088,
+      "rewards/margins": 0.00044798923772759736,
+      "rewards/rejected": -0.00025993891176767647,
       "step": 10
     },
     {
       "epoch": 0.04186289900575615,
-      "grad_norm": 8.872984148971444,
+      "grad_norm": 9.135166094203631,
       "learning_rate": 2.0833333333333333e-07,
-      "logits/chosen": -2.471057653427124,
-      "logits/rejected": -2.376948833465576,
-      "logps/chosen": -296.95635986328125,
-      "logps/rejected": -272.93511962890625,
-      "loss": 0.6917,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.005188685841858387,
-      "rewards/margins": 0.0027170330286026,
-      "rewards/rejected": 0.0024716525804251432,
+      "logits/chosen": -2.2380423545837402,
+      "logits/rejected": -2.1180529594421387,
+      "logps/chosen": -291.8775939941406,
+      "logps/rejected": -266.7220764160156,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": 0.0038302987813949585,
+      "rewards/margins": 0.002486014971509576,
+      "rewards/rejected": 0.0013442838098853827,
       "step": 20
     },
     {
       "epoch": 0.06279434850863422,
-      "grad_norm": 8.9584085433827,
+      "grad_norm": 9.156841604795108,
       "learning_rate": 3.1249999999999997e-07,
-      "logits/chosen": -2.450611114501953,
-      "logits/rejected": -2.3455896377563477,
-      "logps/chosen": -278.2935791015625,
-      "logps/rejected": -238.2044677734375,
-      "loss": 0.6862,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": 0.03269987553358078,
-      "rewards/margins": 0.013539234176278114,
-      "rewards/rejected": 0.019160641357302666,
+      "logits/chosen": -2.204575300216675,
+      "logits/rejected": -2.0733189582824707,
+      "logps/chosen": -273.11627197265625,
+      "logps/rejected": -233.1331024169922,
+      "loss": 0.6858,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.02578217163681984,
+      "rewards/margins": 0.014600785449147224,
+      "rewards/rejected": 0.011181383393704891,
       "step": 30
     },
     {
       "epoch": 0.0837257980115123,
-      "grad_norm": 9.348192963564985,
+      "grad_norm": 9.688914936946615,
       "learning_rate": 4.1666666666666667e-07,
-      "logits/chosen": -2.361204147338867,
-      "logits/rejected": -2.2996718883514404,
-      "logps/chosen": -251.10452270507812,
-      "logps/rejected": -253.61843872070312,
-      "loss": 0.6712,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.04305989295244217,
-      "rewards/margins": 0.048449400812387466,
-      "rewards/rejected": -0.00538951251655817,
+      "logits/chosen": -2.090651750564575,
+      "logits/rejected": -2.000094413757324,
+      "logps/chosen": -247.9547119140625,
+      "logps/rejected": -251.52322387695312,
+      "loss": 0.6676,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.017582237720489502,
+      "rewards/margins": 0.06143052503466606,
+      "rewards/rejected": -0.04384828731417656,
       "step": 40
     },
     {
       "epoch": 0.10465724751439037,
-      "grad_norm": 10.542523800768048,
+      "grad_norm": 11.544993249044957,
       "learning_rate": 4.999731868769026e-07,
-      "logits/chosen": -2.4171504974365234,
-      "logits/rejected": -2.3192880153656006,
-      "logps/chosen": -267.4609375,
-      "logps/rejected": -273.1936950683594,
-      "loss": 0.6466,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": 0.008090761490166187,
-      "rewards/margins": 0.11023788154125214,
-      "rewards/rejected": -0.10214712470769882,
+      "logits/chosen": -2.148649215698242,
+      "logits/rejected": -2.025043249130249,
+      "logps/chosen": -275.79974365234375,
+      "logps/rejected": -283.02716064453125,
+      "loss": 0.6423,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.13076314330101013,
+      "rewards/margins": 0.11921179294586182,
+      "rewards/rejected": -0.24997496604919434,
       "step": 50
     },
     {
       "epoch": 0.12558869701726844,
-      "grad_norm": 19.049765193058207,
+      "grad_norm": 15.350321140435819,
       "learning_rate": 4.990353313429303e-07,
-      "logits/chosen": -2.451169490814209,
-      "logits/rejected": -2.373406410217285,
-      "logps/chosen": -257.0067138671875,
-      "logps/rejected": -271.317626953125,
-      "loss": 0.6146,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -0.11953003704547882,
-      "rewards/margins": 0.22274689376354218,
-      "rewards/rejected": -0.342276930809021,
+      "logits/chosen": -2.2131295204162598,
+      "logits/rejected": -2.1153783798217773,
+      "logps/chosen": -274.0061950683594,
+      "logps/rejected": -293.47113037109375,
+      "loss": 0.616,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.34647348523139954,
+      "rewards/margins": 0.27237194776535034,
+      "rewards/rejected": -0.6188454031944275,
       "step": 60
     },
     {
       "epoch": 0.14652014652014653,
-      "grad_norm": 17.106774713296346,
+      "grad_norm": 17.446151967081736,
       "learning_rate": 4.967625656594781e-07,
-      "logits/chosen": -2.3775253295898438,
-      "logits/rejected": -2.3367419242858887,
-      "logps/chosen": -305.51055908203125,
-      "logps/rejected": -324.1693115234375,
-      "loss": 0.5932,
+      "logits/chosen": -2.153040647506714,
+      "logits/rejected": -2.098363161087036,
+      "logps/chosen": -332.89813232421875,
+      "logps/rejected": -358.3309326171875,
+      "loss": 0.5867,
       "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.2125539779663086,
-      "rewards/margins": 0.3053613603115082,
-      "rewards/rejected": -0.5179153680801392,
+      "rewards/chosen": -0.5407691597938538,
+      "rewards/margins": 0.38723263144493103,
+      "rewards/rejected": -0.9280017614364624,
       "step": 70
     },
     {
       "epoch": 0.1674515960230246,
-      "grad_norm": 22.00744385125915,
+      "grad_norm": 27.42185029612204,
       "learning_rate": 4.93167072587771e-07,
-      "logits/chosen": -2.502345561981201,
-      "logits/rejected": -2.377171277999878,
-      "logps/chosen": -355.249267578125,
-      "logps/rejected": -326.4585876464844,
-      "loss": 0.5855,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.3869813084602356,
-      "rewards/margins": 0.3918095827102661,
-      "rewards/rejected": -0.7787908315658569,
+      "logits/chosen": -2.3443994522094727,
+      "logits/rejected": -2.1753482818603516,
+      "logps/chosen": -361.7871398925781,
+      "logps/rejected": -336.57354736328125,
+      "loss": 0.5807,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.5057138800621033,
+      "rewards/margins": 0.43273383378982544,
+      "rewards/rejected": -0.9384476542472839,
       "step": 80
     },
     {
       "epoch": 0.18838304552590268,
-      "grad_norm": 16.53408344392586,
+      "grad_norm": 22.058501406046492,
       "learning_rate": 4.882681251368548e-07,
-      "logits/chosen": -2.398308277130127,
-      "logits/rejected": -2.338383197784424,
-      "logps/chosen": -301.67816162109375,
-      "logps/rejected": -352.0071716308594,
-      "loss": 0.5635,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.5751432180404663,
-      "rewards/margins": 0.4532225728034973,
-      "rewards/rejected": -1.0283657312393188,
+      "logits/chosen": -2.261949062347412,
+      "logits/rejected": -2.188600778579712,
+      "logps/chosen": -303.0185241699219,
+      "logps/rejected": -365.95440673828125,
+      "loss": 0.5512,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.6372281312942505,
+      "rewards/margins": 0.5772984623908997,
+      "rewards/rejected": -1.2145264148712158,
       "step": 90
     },
     {
       "epoch": 0.20931449502878074,
-      "grad_norm": 20.576427446405972,
+      "grad_norm": 23.10731770703464,
       "learning_rate": 4.820919832540181e-07,
-      "logits/chosen": -2.3533272743225098,
-      "logits/rejected": -2.2900137901306152,
-      "logps/chosen": -348.10467529296875,
-      "logps/rejected": -388.0940246582031,
-      "loss": 0.5631,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.6305854320526123,
-      "rewards/margins": 0.5525855422019958,
-      "rewards/rejected": -1.183171033859253,
+      "logits/chosen": -2.2352817058563232,
+      "logits/rejected": -2.153383255004883,
+      "logps/chosen": -351.9589538574219,
+      "logps/rejected": -398.7929992675781,
+      "loss": 0.5536,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.7218375205993652,
+      "rewards/margins": 0.6266552805900574,
+      "rewards/rejected": -1.3484928607940674,
       "step": 100
     },
     {
       "epoch": 0.2302459445316588,
-      "grad_norm": 20.106003054090923,
+      "grad_norm": 26.246079226878386,
       "learning_rate": 4.7467175306295647e-07,
-      "logits/chosen": -2.4152920246124268,
-      "logits/rejected": -2.345858573913574,
-      "logps/chosen": -330.9453125,
-      "logps/rejected": -357.2662353515625,
-      "loss": 0.575,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.4834519028663635,
-      "rewards/margins": 0.4216728210449219,
-      "rewards/rejected": -0.9051246643066406,
+      "logits/chosen": -2.2616801261901855,
+      "logits/rejected": -2.175260066986084,
+      "logps/chosen": -344.7423400878906,
+      "logps/rejected": -380.02166748046875,
+      "loss": 0.5621,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.6688632369041443,
+      "rewards/margins": 0.5221593976020813,
+      "rewards/rejected": -1.1910226345062256,
       "step": 110
     },
     {
       "epoch": 0.25117739403453687,
-      "grad_norm": 17.544100089861768,
+      "grad_norm": 24.081738347134376,
       "learning_rate": 4.6604720940421207e-07,
-      "logits/chosen": -2.3248140811920166,
-      "logits/rejected": -2.3209547996520996,
-      "logps/chosen": -308.89434814453125,
-      "logps/rejected": -358.4286193847656,
-      "loss": 0.5561,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.4968286156654358,
-      "rewards/margins": 0.5540732145309448,
-      "rewards/rejected": -1.0509017705917358,
+      "logits/chosen": -2.1112895011901855,
+      "logits/rejected": -2.0991122722625732,
+      "logps/chosen": -323.6635437011719,
+      "logps/rejected": -379.61480712890625,
+      "loss": 0.5492,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.6999741792678833,
+      "rewards/margins": 0.6201815009117126,
+      "rewards/rejected": -1.3201555013656616,
       "step": 120
     },
     {
       "epoch": 0.272108843537415,
-      "grad_norm": 17.72276209068045,
+      "grad_norm": 23.369346780966698,
       "learning_rate": 4.5626458262912735e-07,
-      "logits/chosen": -2.3460891246795654,
-      "logits/rejected": -2.315985918045044,
-      "logps/chosen": -341.44708251953125,
-      "logps/rejected": -379.0929260253906,
-      "loss": 0.536,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.7040597200393677,
-      "rewards/margins": 0.48738259077072144,
-      "rewards/rejected": -1.1914422512054443,
+      "logits/chosen": -2.1512370109558105,
+      "logits/rejected": -2.108196973800659,
+      "logps/chosen": -335.7061767578125,
+      "logps/rejected": -375.25469970703125,
+      "loss": 0.5373,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.7045310139656067,
+      "rewards/margins": 0.5020760297775269,
+      "rewards/rejected": -1.2066071033477783,
       "step": 130
     },
     {
       "epoch": 0.29304029304029305,
-      "grad_norm": 21.32563247174665,
+      "grad_norm": 20.81429209370454,
       "learning_rate": 4.453763107901675e-07,
-      "logits/chosen": -2.328991174697876,
-      "logits/rejected": -2.274400472640991,
-      "logps/chosen": -354.708984375,
-      "logps/rejected": -377.31011962890625,
-      "loss": 0.5535,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -0.506158173084259,
-      "rewards/margins": 0.6448914408683777,
-      "rewards/rejected": -1.1510496139526367,
+      "logits/chosen": -2.191638469696045,
+      "logits/rejected": -2.123680591583252,
+      "logps/chosen": -364.34161376953125,
+      "logps/rejected": -393.08013916015625,
+      "loss": 0.5473,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.6576336622238159,
+      "rewards/margins": 0.6938663125038147,
+      "rewards/rejected": -1.3515000343322754,
       "step": 140
     },
     {
       "epoch": 0.3139717425431711,
-      "grad_norm": 25.439845700518678,
+      "grad_norm": 29.1079724061151,
       "learning_rate": 4.3344075855595097e-07,
-      "logits/chosen": -2.2812905311584473,
-      "logits/rejected": -2.228199005126953,
-      "logps/chosen": -332.77044677734375,
-      "logps/rejected": -350.35162353515625,
-      "loss": 0.5502,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.6236374974250793,
-      "rewards/margins": 0.47987040877342224,
-      "rewards/rejected": -1.1035077571868896,
+      "logits/chosen": -2.207198143005371,
+      "logits/rejected": -2.1470441818237305,
+      "logps/chosen": -336.94122314453125,
+      "logps/rejected": -356.14520263671875,
+      "loss": 0.5473,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.7152302861213684,
+      "rewards/margins": 0.504962682723999,
+      "rewards/rejected": -1.2201929092407227,
       "step": 150
     },
     {
       "epoch": 0.3349031920460492,
-      "grad_norm": 22.437229700307615,
+      "grad_norm": 27.917087192433524,
       "learning_rate": 4.2052190435769554e-07,
-      "logits/chosen": -2.3055479526519775,
-      "logits/rejected": -2.2253096103668213,
-      "logps/chosen": -341.20880126953125,
-      "logps/rejected": -384.2608337402344,
-      "loss": 0.5544,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.743469774723053,
-      "rewards/margins": 0.6333221793174744,
-      "rewards/rejected": -1.3767919540405273,
+      "logits/chosen": -2.2518768310546875,
+      "logits/rejected": -2.1720962524414062,
+      "logps/chosen": -333.5887756347656,
+      "logps/rejected": -383.0411682128906,
+      "loss": 0.5556,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.7204815745353699,
+      "rewards/margins": 0.7067698240280151,
+      "rewards/rejected": -1.4272514581680298,
       "step": 160
     },
     {
       "epoch": 0.35583464154892724,
-      "grad_norm": 22.375265306667423,
+      "grad_norm": 28.43046248619016,
       "learning_rate": 4.0668899744407567e-07,
-      "logits/chosen": -2.170342206954956,
-      "logits/rejected": -2.1414666175842285,
-      "logps/chosen": -315.53131103515625,
-      "logps/rejected": -349.3824462890625,
-      "loss": 0.5418,
+      "logits/chosen": -2.0816593170166016,
+      "logits/rejected": -2.050276041030884,
+      "logps/chosen": -338.7085876464844,
+      "logps/rejected": -370.1266174316406,
+      "loss": 0.5389,
       "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.647168755531311,
-      "rewards/margins": 0.554858386516571,
-      "rewards/rejected": -1.2020272016525269,
+      "rewards/chosen": -0.9299048185348511,
+      "rewards/margins": 0.5225407481193542,
+      "rewards/rejected": -1.45244562625885,
       "step": 170
     },
     {
       "epoch": 0.37676609105180536,
-      "grad_norm": 22.685803060689793,
+      "grad_norm": 27.829977826652623,
       "learning_rate": 3.920161866827889e-07,
-      "logits/chosen": -2.2532403469085693,
-      "logits/rejected": -2.2120420932769775,
-      "logps/chosen": -348.15325927734375,
-      "logps/rejected": -376.5264587402344,
-      "loss": 0.5167,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.8467988967895508,
-      "rewards/margins": 0.5318444967269897,
-      "rewards/rejected": -1.3786433935165405,
+      "logits/chosen": -2.0916779041290283,
+      "logits/rejected": -2.0410423278808594,
+      "logps/chosen": -364.9996643066406,
+      "logps/rejected": -400.76239013671875,
+      "loss": 0.5074,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.081892967224121,
+      "rewards/margins": 0.6016584038734436,
+      "rewards/rejected": -1.6835514307022095,
       "step": 180
     },
     {
       "epoch": 0.3976975405546834,
-      "grad_norm": 23.346233993869507,
+      "grad_norm": 27.60089876506741,
       "learning_rate": 3.765821230985757e-07,
-      "logits/chosen": -2.181307315826416,
-      "logits/rejected": -2.1494665145874023,
-      "logps/chosen": -329.13287353515625,
-      "logps/rejected": -377.4722595214844,
-      "loss": 0.5433,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.7779133915901184,
-      "rewards/margins": 0.5419718027114868,
-      "rewards/rejected": -1.319885015487671,
+      "logits/chosen": -2.0626139640808105,
+      "logits/rejected": -2.0298991203308105,
+      "logps/chosen": -340.3715515136719,
+      "logps/rejected": -395.8924255371094,
+      "loss": 0.5326,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.9407373666763306,
+      "rewards/margins": 0.6121485829353333,
+      "rewards/rejected": -1.5528860092163086,
       "step": 190
     },
     {
       "epoch": 0.4186289900575615,
-      "grad_norm": 59.244174530672744,
+      "grad_norm": 38.5217107381582,
       "learning_rate": 3.604695382782159e-07,
-      "logits/chosen": -2.183567762374878,
-      "logits/rejected": -2.1823012828826904,
-      "logps/chosen": -307.64337158203125,
-      "logps/rejected": -381.5365905761719,
-      "loss": 0.5493,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.6649466753005981,
-      "rewards/margins": 0.5270031094551086,
-      "rewards/rejected": -1.1919496059417725,
+      "logits/chosen": -2.0065114498138428,
+      "logits/rejected": -2.0017216205596924,
+      "logps/chosen": -332.4302673339844,
+      "logps/rejected": -414.33251953125,
+      "loss": 0.5384,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.9781379699707031,
+      "rewards/margins": 0.6036709547042847,
+      "rewards/rejected": -1.5818089246749878,
       "step": 200
     },
     {
       "epoch": 0.43956043956043955,
-      "grad_norm": 21.4577312447254,
+      "grad_norm": 25.593502267183467,
       "learning_rate": 3.4376480090239047e-07,
-      "logits/chosen": -2.2622885704040527,
-      "logits/rejected": -2.175058603286743,
-      "logps/chosen": -374.94775390625,
-      "logps/rejected": -381.5616760253906,
-      "loss": 0.5386,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.7791308164596558,
-      "rewards/margins": 0.5941972732543945,
-      "rewards/rejected": -1.3733280897140503,
+      "logits/chosen": -2.039764881134033,
+      "logits/rejected": -1.9067729711532593,
+      "logps/chosen": -379.778076171875,
+      "logps/rejected": -392.64239501953125,
+      "loss": 0.5337,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.8867334127426147,
+      "rewards/margins": 0.6549147367477417,
+      "rewards/rejected": -1.5416481494903564,
       "step": 210
     },
     {
       "epoch": 0.4604918890633176,
-      "grad_norm": 27.605297631673444,
+      "grad_norm": 36.182958654851035,
       "learning_rate": 3.265574537815398e-07,
-      "logits/chosen": -2.142841339111328,
-      "logits/rejected": -2.153688907623291,
-      "logps/chosen": -322.2380065917969,
-      "logps/rejected": -390.0839538574219,
-      "loss": 0.5304,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.9062248468399048,
-      "rewards/margins": 0.6386878490447998,
-      "rewards/rejected": -1.544912576675415,
+      "logits/chosen": -1.8658435344696045,
+      "logits/rejected": -1.8497034311294556,
+      "logps/chosen": -316.08392333984375,
+      "logps/rejected": -390.88018798828125,
+      "loss": 0.5137,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.9008970260620117,
+      "rewards/margins": 0.7025474309921265,
+      "rewards/rejected": -1.6034444570541382,
       "step": 220
     },
     {
       "epoch": 0.48142333856619574,
-      "grad_norm": 25.98305182876635,
+      "grad_norm": 30.893691078025817,
       "learning_rate": 3.0893973387735683e-07,
-      "logits/chosen": -2.246525526046753,
-      "logits/rejected": -2.182516098022461,
-      "logps/chosen": -357.104736328125,
-      "logps/rejected": -429.77337646484375,
-      "loss": 0.5391,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.0580464601516724,
-      "rewards/margins": 0.896924614906311,
-      "rewards/rejected": -1.9549709558486938,
+      "logits/chosen": -1.9850623607635498,
+      "logits/rejected": -1.8835790157318115,
+      "logps/chosen": -350.849853515625,
+      "logps/rejected": -432.809326171875,
+      "loss": 0.5227,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.0541720390319824,
+      "rewards/margins": 0.9999459385871887,
+      "rewards/rejected": -2.0541179180145264,
       "step": 230
     },
     {
       "epoch": 0.5023547880690737,
-      "grad_norm": 21.13667543588383,
+      "grad_norm": 33.02799444972093,
       "learning_rate": 2.910060778827554e-07,
-      "logits/chosen": -2.2703707218170166,
-      "logits/rejected": -2.204939126968384,
-      "logps/chosen": -368.1454162597656,
-      "logps/rejected": -415.8702697753906,
-      "loss": 0.5053,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.9035312533378601,
-      "rewards/margins": 0.7343918085098267,
-      "rewards/rejected": -1.637923002243042,
+      "logits/chosen": -1.9802770614624023,
+      "logits/rejected": -1.880558967590332,
+      "logps/chosen": -384.36138916015625,
+      "logps/rejected": -446.0873107910156,
+      "loss": 0.4943,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.1203218698501587,
+      "rewards/margins": 0.8701934814453125,
+      "rewards/rejected": -1.9905153512954712,
       "step": 240
     },
     {
       "epoch": 0.5232862375719518,
-      "grad_norm": 25.84731298493337,
+      "grad_norm": 35.534978981621464,
       "learning_rate": 2.7285261601056697e-07,
-      "logits/chosen": -2.19517183303833,
-      "logits/rejected": -2.1099765300750732,
-      "logps/chosen": -365.8097229003906,
-      "logps/rejected": -419.41497802734375,
-      "loss": 0.5115,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -0.9770700335502625,
-      "rewards/margins": 0.9047032594680786,
-      "rewards/rejected": -1.8817732334136963,
+      "logits/chosen": -1.9159233570098877,
+      "logits/rejected": -1.778590202331543,
+      "logps/chosen": -370.39093017578125,
+      "logps/rejected": -430.15338134765625,
+      "loss": 0.5013,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -1.059476375579834,
+      "rewards/margins": 0.9878945350646973,
+      "rewards/rejected": -2.0473709106445312,
       "step": 250
     },
     {
       "epoch": 0.54421768707483,
-      "grad_norm": 30.85559229261849,
+      "grad_norm": 32.532057976810556,
       "learning_rate": 2.5457665670441937e-07,
-      "logits/chosen": -1.9430633783340454,
-      "logits/rejected": -1.9308143854141235,
-      "logps/chosen": -361.0708923339844,
-      "logps/rejected": -423.67242431640625,
-      "loss": 0.5074,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -1.0502084493637085,
-      "rewards/margins": 0.7874974012374878,
-      "rewards/rejected": -1.8377059698104858,
+      "logits/chosen": -2.016502857208252,
+      "logits/rejected": -2.0029537677764893,
+      "logps/chosen": -336.88287353515625,
+      "logps/rejected": -397.57806396484375,
+      "loss": 0.5046,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.8760363459587097,
+      "rewards/margins": 0.7554494738578796,
+      "rewards/rejected": -1.631486177444458,
       "step": 260
     },
     {
       "epoch": 0.565149136577708,
-      "grad_norm": 24.551312599118287,
+      "grad_norm": 29.326160875282234,
       "learning_rate": 2.3627616503391812e-07,
-      "logits/chosen": -1.6331418752670288,
-      "logits/rejected": -1.550295352935791,
-      "logps/chosen": -384.88031005859375,
-      "logps/rejected": -451.5082092285156,
-      "loss": 0.5198,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.0247470140457153,
-      "rewards/margins": 0.8930918574333191,
-      "rewards/rejected": -1.9178390502929688,
+      "logits/chosen": -2.0074622631073,
+      "logits/rejected": -1.963415503501892,
+      "logps/chosen": -375.93359375,
+      "logps/rejected": -444.92010498046875,
+      "loss": 0.5195,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.9895309209823608,
+      "rewards/margins": 0.921257495880127,
+      "rewards/rejected": -1.9107885360717773,
       "step": 270
     },
     {
       "epoch": 0.5860805860805861,
-      "grad_norm": 29.74692013409924,
+      "grad_norm": 31.08862280431646,
       "learning_rate": 2.1804923757009882e-07,
-      "logits/chosen": -1.4030331373214722,
-      "logits/rejected": -1.2896840572357178,
-      "logps/chosen": -341.4692077636719,
-      "logps/rejected": -389.22454833984375,
-      "loss": 0.5213,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.9064887762069702,
-      "rewards/margins": 0.7257590293884277,
-      "rewards/rejected": -1.6322479248046875,
+      "logits/chosen": -1.8468096256256104,
+      "logits/rejected": -1.7995507717132568,
+      "logps/chosen": -347.49713134765625,
+      "logps/rejected": -392.85498046875,
+      "loss": 0.5164,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.0229954719543457,
+      "rewards/margins": 0.7037075757980347,
+      "rewards/rejected": -1.7267030477523804,
       "step": 280
     },
     {
       "epoch": 0.6070120355834642,
-      "grad_norm": 26.977158157494642,
+      "grad_norm": 35.420244394607494,
       "learning_rate": 1.9999357655598891e-07,
-      "logits/chosen": -1.4508934020996094,
-      "logits/rejected": -1.3269082307815552,
-      "logps/chosen": -345.9167175292969,
-      "logps/rejected": -418.6991271972656,
-      "loss": 0.5151,
+      "logits/chosen": -1.9150333404541016,
+      "logits/rejected": -1.819837212562561,
+      "logps/chosen": -338.6258544921875,
+      "logps/rejected": -407.5479431152344,
+      "loss": 0.5104,
       "rewards/accuracies": 0.78125,
-      "rewards/chosen": -1.0557668209075928,
-      "rewards/margins": 0.7933701276779175,
-      "rewards/rejected": -1.8491369485855103,
+      "rewards/chosen": -1.0374407768249512,
+      "rewards/margins": 0.7471441030502319,
+      "rewards/rejected": -1.7845849990844727,
       "step": 290
     },
     {
       "epoch": 0.6279434850863422,
-      "grad_norm": 25.588639007041994,
+      "grad_norm": 29.946282422909608,
       "learning_rate": 1.8220596619089573e-07,
-      "logits/chosen": -1.3906519412994385,
-      "logits/rejected": -1.234431505203247,
-      "logps/chosen": -424.56378173828125,
-      "logps/rejected": -464.31463623046875,
-      "loss": 0.488,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.1725590229034424,
-      "rewards/margins": 0.8173145055770874,
-      "rewards/rejected": -1.9898736476898193,
+      "logits/chosen": -1.805354356765747,
+      "logits/rejected": -1.706356406211853,
+      "logps/chosen": -411.9443359375,
+      "logps/rejected": -455.328857421875,
+      "loss": 0.4834,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.102001667022705,
+      "rewards/margins": 0.8499504327774048,
+      "rewards/rejected": -1.9519522190093994,
       "step": 300
     },
     {
       "epoch": 0.6488749345892203,
-      "grad_norm": 35.84688001545634,
+      "grad_norm": 33.48721686835146,
       "learning_rate": 1.647817538357072e-07,
-      "logits/chosen": -1.2901729345321655,
-      "logits/rejected": -1.141137719154358,
-      "logps/chosen": -441.1419982910156,
-      "logps/rejected": -498.540283203125,
+      "logits/chosen": -1.8414630889892578,
+      "logits/rejected": -1.7567808628082275,
+      "logps/chosen": -416.7355041503906,
+      "logps/rejected": -473.2708435058594,
       "loss": 0.4995,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -1.4990507364273071,
-      "rewards/margins": 1.0634124279022217,
-      "rewards/rejected": -2.5624632835388184,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.3046259880065918,
+      "rewards/margins": 1.056304931640625,
+      "rewards/rejected": -2.360930919647217,
       "step": 310
     },
     {
       "epoch": 0.6698063840920984,
-      "grad_norm": 37.73844743050267,
+      "grad_norm": 33.90432965296341,
       "learning_rate": 1.478143389201113e-07,
-      "logits/chosen": -1.053849458694458,
-      "logits/rejected": -0.8557470440864563,
-      "logps/chosen": -397.40447998046875,
-      "logps/rejected": -475.0392150878906,
-      "loss": 0.4813,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -1.461068868637085,
-      "rewards/margins": 1.0757315158843994,
-      "rewards/rejected": -2.5368001461029053,
+      "logits/chosen": -1.8603925704956055,
+      "logits/rejected": -1.7522039413452148,
+      "logps/chosen": -358.3260498046875,
+      "logps/rejected": -422.6536560058594,
+      "loss": 0.4826,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.1275660991668701,
+      "rewards/margins": 0.9410696029663086,
+      "rewards/rejected": -2.0686357021331787,
       "step": 320
     },
     {
       "epoch": 0.6907378335949764,
-      "grad_norm": 26.325601270856676,
+      "grad_norm": 27.513905071763368,
       "learning_rate": 1.3139467229135998e-07,
-      "logits/chosen": -1.1438688039779663,
-      "logits/rejected": -1.1061335802078247,
-      "logps/chosen": -375.85955810546875,
-      "logps/rejected": -456.5414123535156,
-      "loss": 0.4993,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -1.1525086164474487,
-      "rewards/margins": 0.8357551693916321,
-      "rewards/rejected": -1.988263726234436,
+      "logits/chosen": -1.901578664779663,
+      "logits/rejected": -1.8754088878631592,
+      "logps/chosen": -363.4566345214844,
+      "logps/rejected": -448.67803955078125,
+      "loss": 0.4893,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.0841090679168701,
+      "rewards/margins": 0.8770705461502075,
+      "rewards/rejected": -1.9611797332763672,
       "step": 330
     },
     {
       "epoch": 0.7116692830978545,
-      "grad_norm": 37.090497017079386,
+      "grad_norm": 39.229104335542225,
       "learning_rate": 1.1561076868822755e-07,
-      "logits/chosen": -0.9682319760322571,
-      "logits/rejected": -0.8497031927108765,
-      "logps/chosen": -403.8167419433594,
-      "logps/rejected": -476.2872009277344,
-      "loss": 0.4848,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -1.249403953552246,
-      "rewards/margins": 0.9320135116577148,
-      "rewards/rejected": -2.181417226791382,
+      "logits/chosen": -1.7680823802947998,
+      "logits/rejected": -1.7113357782363892,
+      "logps/chosen": -401.6238708496094,
+      "logps/rejected": -475.1571350097656,
+      "loss": 0.4905,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.276782751083374,
+      "rewards/margins": 0.9465486407279968,
+      "rewards/rejected": -2.2233314514160156,
       "step": 340
     },
     {
       "epoch": 0.7326007326007326,
-      "grad_norm": 29.0205923314937,
+      "grad_norm": 31.94987608162002,
       "learning_rate": 1.0054723495346482e-07,
-      "logits/chosen": -0.8897636532783508,
-      "logits/rejected": -0.6620567440986633,
-      "logps/chosen": -401.22882080078125,
-      "logps/rejected": -475.3289489746094,
-      "loss": 0.4573,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -1.493096113204956,
-      "rewards/margins": 0.9968076944351196,
-      "rewards/rejected": -2.489903688430786,
+      "logits/chosen": -1.806357979774475,
+      "logits/rejected": -1.7052780389785767,
+      "logps/chosen": -360.28607177734375,
+      "logps/rejected": -427.3250427246094,
+      "loss": 0.4608,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.143326759338379,
+      "rewards/margins": 0.9258495569229126,
+      "rewards/rejected": -2.069176197052002,
       "step": 350
     },
     {
       "epoch": 0.7535321821036107,
-      "grad_norm": 28.465041930216806,
+      "grad_norm": 28.62799992820538,
       "learning_rate": 8.628481651367875e-08,
-      "logits/chosen": -0.8191855549812317,
-      "logits/rejected": -0.5711785554885864,
-      "logps/chosen": -448.5098571777344,
-      "logps/rejected": -526.4882202148438,
-      "loss": 0.5036,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -1.5041682720184326,
-      "rewards/margins": 1.1857599020004272,
-      "rewards/rejected": -2.6899282932281494,
+      "logits/chosen": -1.7983932495117188,
+      "logits/rejected": -1.6765844821929932,
+      "logps/chosen": -408.5673522949219,
+      "logps/rejected": -467.3724060058594,
+      "loss": 0.5179,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.1572364568710327,
+      "rewards/margins": 0.9978920817375183,
+      "rewards/rejected": -2.1551289558410645,
       "step": 360
     },
     {
       "epoch": 0.7744636316064888,
-      "grad_norm": 29.111893282608257,
+      "grad_norm": 31.352439028541415,
       "learning_rate": 7.289996455765748e-08,
-      "logits/chosen": -0.8937414288520813,
-      "logits/rejected": -0.7008529901504517,
-      "logps/chosen": -409.792724609375,
-      "logps/rejected": -478.1075134277344,
-      "loss": 0.5009,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -1.5043065547943115,
-      "rewards/margins": 0.9667744636535645,
-      "rewards/rejected": -2.471081018447876,
+      "logits/chosen": -1.8380626440048218,
+      "logits/rejected": -1.7419719696044922,
+      "logps/chosen": -364.7669982910156,
+      "logps/rejected": -425.0555114746094,
+      "loss": 0.5005,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.1147412061691284,
+      "rewards/margins": 0.8857861757278442,
+      "rewards/rejected": -2.0005273818969727,
       "step": 370
     },
     {
       "epoch": 0.7953950811093669,
-      "grad_norm": 29.70871758620533,
+      "grad_norm": 26.28282670785366,
       "learning_rate": 6.046442623320145e-08,
-      "logits/chosen": -0.675798237323761,
-      "logits/rejected": -0.5979479551315308,
-      "logps/chosen": -405.40380859375,
-      "logps/rejected": -542.0068969726562,
-      "loss": 0.4974,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -1.6684430837631226,
-      "rewards/margins": 1.1851465702056885,
-      "rewards/rejected": -2.8535895347595215,
+      "logits/chosen": -1.6995561122894287,
+      "logits/rejected": -1.672106146812439,
+      "logps/chosen": -367.4508361816406,
+      "logps/rejected": -488.54913330078125,
+      "loss": 0.4978,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.3325788974761963,
+      "rewards/margins": 1.0493899583816528,
+      "rewards/rejected": -2.3819689750671387,
       "step": 380
     },
     {
       "epoch": 0.8163265306122449,
-      "grad_norm": 36.269601768258234,
+      "grad_norm": 29.780729258634107,
       "learning_rate": 4.904486005914027e-08,
-      "logits/chosen": -0.9742690920829773,
-      "logits/rejected": -0.7953445315361023,
-      "logps/chosen": -488.8130798339844,
-      "logps/rejected": -539.947021484375,
-      "loss": 0.4893,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -1.606419324874878,
-      "rewards/margins": 0.8702549934387207,
-      "rewards/rejected": -2.4766743183135986,
+      "logits/chosen": -1.8319686651229858,
+      "logits/rejected": -1.7405626773834229,
+      "logps/chosen": -448.5791931152344,
+      "logps/rejected": -498.88104248046875,
+      "loss": 0.4845,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.2597005367279053,
+      "rewards/margins": 0.8696678876876831,
+      "rewards/rejected": -2.129368543624878,
       "step": 390
     },
     {
       "epoch": 0.837257980115123,
-      "grad_norm": 30.05583737574385,
+      "grad_norm": 30.094371408831872,
       "learning_rate": 3.8702478614051345e-08,
-      "logits/chosen": -0.8497405052185059,
-      "logits/rejected": -0.6598803400993347,
-      "logps/chosen": -398.9614562988281,
-      "logps/rejected": -482.28302001953125,
-      "loss": 0.5016,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.5102481842041016,
-      "rewards/margins": 0.9544359445571899,
-      "rewards/rejected": -2.464684009552002,
+      "logits/chosen": -1.7541265487670898,
+      "logits/rejected": -1.6716575622558594,
+      "logps/chosen": -365.5286560058594,
+      "logps/rejected": -440.4735412597656,
+      "loss": 0.5052,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.2250964641571045,
+      "rewards/margins": 0.877382755279541,
+      "rewards/rejected": -2.1024792194366455,
       "step": 400
     },
     {
       "epoch": 0.858189429618001,
-      "grad_norm": 27.172370132213754,
+      "grad_norm": 32.623663420886714,
       "learning_rate": 2.9492720416985e-08,
-      "logits/chosen": -1.1038849353790283,
-      "logits/rejected": -0.9001197814941406,
-      "logps/chosen": -444.0462951660156,
-      "logps/rejected": -514.1170654296875,
-      "loss": 0.5039,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -1.5119292736053467,
-      "rewards/margins": 1.0207693576812744,
-      "rewards/rejected": -2.5326988697052,
+      "logits/chosen": -1.8887017965316772,
+      "logits/rejected": -1.79939866065979,
+      "logps/chosen": -404.93719482421875,
+      "logps/rejected": -470.50714111328125,
+      "loss": 0.4989,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.173086404800415,
+      "rewards/margins": 0.9763132333755493,
+      "rewards/rejected": -2.149399518966675,
       "step": 410
     },
     {
       "epoch": 0.8791208791208791,
-      "grad_norm": 30.512211155868616,
+      "grad_norm": 32.9180383847494,
       "learning_rate": 2.1464952759020856e-08,
-      "logits/chosen": -0.9595249891281128,
-      "logits/rejected": -0.8507660627365112,
-      "logps/chosen": -405.39862060546875,
-      "logps/rejected": -510.281494140625,
-      "loss": 0.4905,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -1.5675805807113647,
-      "rewards/margins": 1.0063538551330566,
-      "rewards/rejected": -2.573934555053711,
+      "logits/chosen": -1.7282394170761108,
+      "logits/rejected": -1.6893911361694336,
+      "logps/chosen": -366.7571716308594,
+      "logps/rejected": -468.3814392089844,
+      "loss": 0.4868,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.2342027425765991,
+      "rewards/margins": 0.9661720991134644,
+      "rewards/rejected": -2.2003750801086426,
       "step": 420
     },
     {
       "epoch": 0.9000523286237572,
-      "grad_norm": 28.009936036028865,
+      "grad_norm": 33.302644642921706,
       "learning_rate": 1.4662207078575684e-08,
-      "logits/chosen": -0.9552485346794128,
-      "logits/rejected": -0.760822057723999,
-      "logps/chosen": -430.9397888183594,
-      "logps/rejected": -507.698486328125,
-      "loss": 0.4751,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -1.497207522392273,
-      "rewards/margins": 0.9993329048156738,
-      "rewards/rejected": -2.4965405464172363,
+      "logits/chosen": -1.7425053119659424,
+      "logits/rejected": -1.6233768463134766,
+      "logps/chosen": -397.09466552734375,
+      "logps/rejected": -470.15692138671875,
+      "loss": 0.4755,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.2036662101745605,
+      "rewards/margins": 0.977012038230896,
+      "rewards/rejected": -2.180678129196167,
       "step": 430
     },
     {
       "epoch": 0.9209837781266352,
-      "grad_norm": 29.996752006101683,
+      "grad_norm": 30.223398646627228,
       "learning_rate": 9.12094829893642e-09,
-      "logits/chosen": -1.0940220355987549,
-      "logits/rejected": -0.9033029675483704,
-      "logps/chosen": -399.84954833984375,
-      "logps/rejected": -462.63153076171875,
-      "loss": 0.4862,
+      "logits/chosen": -1.86935555934906,
+      "logits/rejected": -1.7585684061050415,
+      "logps/chosen": -369.0350341796875,
+      "logps/rejected": -429.52020263671875,
+      "loss": 0.4945,
       "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.4471704959869385,
-      "rewards/margins": 0.9819791913032532,
-      "rewards/rejected": -2.429149866104126,
+      "rewards/chosen": -1.1897361278533936,
+      "rewards/margins": 0.9693399667739868,
+      "rewards/rejected": -2.15907621383667,
       "step": 440
     },
     {
       "epoch": 0.9419152276295133,
-      "grad_norm": 30.711855612709723,
+      "grad_norm": 32.3520965832113,
       "learning_rate": 4.8708793644441086e-09,
-      "logits/chosen": -0.8034582138061523,
-      "logits/rejected": -0.6741777658462524,
-      "logps/chosen": -414.15447998046875,
-      "logps/rejected": -519.9739990234375,
-      "loss": 0.4883,
+      "logits/chosen": -1.600304365158081,
+      "logits/rejected": -1.5472590923309326,
+      "logps/chosen": -384.4742431640625,
+      "logps/rejected": -480.8585510253906,
+      "loss": 0.4854,
       "rewards/accuracies": 0.78125,
-      "rewards/chosen": -1.5021246671676636,
-      "rewards/margins": 1.1551309823989868,
-      "rewards/rejected": -2.6572554111480713,
+      "rewards/chosen": -1.2649081945419312,
+      "rewards/margins": 1.0646650791168213,
+      "rewards/rejected": -2.329573154449463,
       "step": 450
     },
     {
       "epoch": 0.9628466771323915,
-      "grad_norm": 40.76374555079479,
+      "grad_norm": 40.57863055733759,
       "learning_rate": 1.9347820230782295e-09,
-      "logits/chosen": -0.977044403553009,
-      "logits/rejected": -0.7038423418998718,
-      "logps/chosen": -423.5811462402344,
-      "logps/rejected": -475.4755859375,
-      "loss": 0.4981,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.5915157794952393,
-      "rewards/margins": 0.929116427898407,
-      "rewards/rejected": -2.520632266998291,
+      "logits/chosen": -1.7524124383926392,
+      "logits/rejected": -1.6150585412979126,
+      "logps/chosen": -385.82073974609375,
+      "logps/rejected": -439.78973388671875,
+      "loss": 0.4974,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.265455722808838,
+      "rewards/margins": 0.9535617828369141,
+      "rewards/rejected": -2.219017505645752,
       "step": 460
     },
     {
       "epoch": 0.9837781266352695,
-      "grad_norm": 26.303634709848364,
+      "grad_norm": 26.449373393329658,
       "learning_rate": 3.2839470889836627e-10,
-      "logits/chosen": -0.9590628743171692,
-      "logits/rejected": -0.8002158403396606,
-      "logps/chosen": -434.5794982910156,
-      "logps/rejected": -501.19775390625,
-      "loss": 0.4747,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.4538209438323975,
-      "rewards/margins": 0.9469608068466187,
-      "rewards/rejected": -2.4007816314697266,
+      "logits/chosen": -1.7558813095092773,
+      "logits/rejected": -1.6784464120864868,
+      "logps/chosen": -399.96173095703125,
+      "logps/rejected": -463.38787841796875,
+      "loss": 0.4838,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.169880747795105,
+      "rewards/margins": 0.9056078195571899,
+      "rewards/rejected": -2.075488567352295,
       "step": 470
     },
     {
       "epoch": 0.9984301412872841,
-      "eval_logits/chosen": -0.8246170282363892,
-      "eval_logits/rejected": -0.6947768330574036,
-      "eval_logps/chosen": -401.7669677734375,
-      "eval_logps/rejected": -508.43353271484375,
-      "eval_loss": 0.4935465157032013,
+      "eval_logits/chosen": -1.7399183511734009,
+      "eval_logits/rejected": -1.6741559505462646,
+      "eval_logps/chosen": -363.3537902832031,
+      "eval_logps/rejected": -469.47021484375,
+      "eval_loss": 0.49291545152664185,
       "eval_rewards/accuracies": 0.78515625,
-      "eval_rewards/chosen": -1.4744462966918945,
-      "eval_rewards/margins": 1.0516830682754517,
-      "eval_rewards/rejected": -2.5261292457580566,
-      "eval_runtime": 167.9754,
-      "eval_samples_per_second": 11.907,
-      "eval_steps_per_second": 0.191,
+      "eval_rewards/chosen": -1.1416336297988892,
+      "eval_rewards/margins": 1.0502383708953857,
+      "eval_rewards/rejected": -2.1918721199035645,
+      "eval_runtime": 168.8714,
+      "eval_samples_per_second": 11.843,
+      "eval_steps_per_second": 0.189,
       "step": 477
     },
     {
       "epoch": 0.9984301412872841,
       "step": 477,
       "total_flos": 0.0,
-      "train_loss": 0.537537020957195,
-      "train_runtime": 15566.6343,
-      "train_samples_per_second": 3.927,
-      "train_steps_per_second": 0.031
+      "train_loss": 0.5343590847351266,
+      "train_runtime": 13524.9429,
+      "train_samples_per_second": 4.52,
+      "train_steps_per_second": 0.035
     }
   ],
   "logging_steps": 10,