Model save

README.md

@@ -16,15 +16,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model was trained from scratch on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.5260
-- Rewards/chosen: -0.6381
-- Rewards/rejected: -1.4215
-- Rewards/accuracies: 0.7773
-- Rewards/margins: 0.7834
-- Logps/rejected: -409.2955
-- Logps/chosen: -334.1724
-- Logits/rejected: -0.8835
-- Logits/chosen: -1.0303
+- Loss: 0.5279
+- Rewards/chosen: -0.6819
+- Rewards/rejected: -1.4900
+- Rewards/accuracies: 0.7812
+- Rewards/margins: 0.8081
+- Logps/rejected: -425.1121
+- Logps/chosen: -348.1232
+- Logits/rejected: -1.3790
+- Logits/chosen: -1.4815
 
 ## Model description
 
@@ -61,7 +61,7 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.5174        | 0.9984 | 477  | 0.5260          | -0.6381        | -1.4215          | 0.7773             | 0.7834          | -409.2955      | -334.1724    | -0.8835         | -1.0303       |
+| 0.5147        | 0.9984 | 477  | 0.5279          | -0.6819        | -1.4900          | 0.7812             | 0.8081          | -425.1121      | -348.1232    | -1.3790         | -1.4815       |
 
 
 ### Framework versions

all_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.9984301412872841,
     "total_flos": 0.0,
-    "train_loss": 0.5633771029658288,
-    "train_runtime": 15455.4899,
+    "train_loss": 0.5669088098737929,
+    "train_runtime": 15771.2037,
     "train_samples": 61134,
-    "train_samples_per_second": 3.955,
-    "train_steps_per_second": 0.031
+    "train_samples_per_second": 3.876,
+    "train_steps_per_second": 0.03
 }

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.9984301412872841,
     "total_flos": 0.0,
-    "train_loss": 0.5633771029658288,
-    "train_runtime": 15455.4899,
+    "train_loss": 0.5669088098737929,
+    "train_runtime": 15771.2037,
     "train_samples": 61134,
-    "train_samples_per_second": 3.955,
-    "train_steps_per_second": 0.031
+    "train_samples_per_second": 3.876,
+    "train_steps_per_second": 0.03
 }

trainer_state.json

@@ -10,12 +10,12 @@
   "log_history": [
     {
       "epoch": 0.0020931449502878076,
-      "grad_norm": 11.915830605029264,
+      "grad_norm": 9.911216937670318,
       "learning_rate": 1.0416666666666666e-08,
-      "logits/chosen": -2.900132894515991,
-      "logits/rejected": -2.834955930709839,
-      "logps/chosen": -317.546875,
-      "logps/rejected": -362.03985595703125,
+      "logits/chosen": -2.8090171813964844,
+      "logits/rejected": -2.7643635272979736,
+      "logps/chosen": -333.44940185546875,
+      "logps/rejected": -378.9651184082031,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -25,733 +25,733 @@
     },
     {
       "epoch": 0.020931449502878074,
-      "grad_norm": 9.037023118445425,
+      "grad_norm": 9.008729965360628,
       "learning_rate": 1.0416666666666667e-07,
-      "logits/chosen": -2.636439800262451,
-      "logits/rejected": -2.5899064540863037,
-      "logps/chosen": -314.6423645019531,
-      "logps/rejected": -281.7502136230469,
-      "loss": 0.6931,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.0006509354570880532,
-      "rewards/margins": 0.00030382387922145426,
-      "rewards/rejected": 0.0003471115487627685,
+      "logits/chosen": -2.597419261932373,
+      "logits/rejected": -2.561866521835327,
+      "logps/chosen": -323.94671630859375,
+      "logps/rejected": -288.5050048828125,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.3819444477558136,
+      "rewards/chosen": 0.0003000612196046859,
+      "rewards/margins": -0.00015054795949254185,
+      "rewards/rejected": 0.00045060913544148207,
       "step": 10
     },
     {
       "epoch": 0.04186289900575615,
-      "grad_norm": 8.350785008747597,
+      "grad_norm": 7.902884214874011,
       "learning_rate": 2.0833333333333333e-07,
-      "logits/chosen": -2.7466042041778564,
-      "logits/rejected": -2.6606907844543457,
-      "logps/chosen": -315.2346496582031,
-      "logps/rejected": -285.74896240234375,
-      "loss": 0.6927,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.0006443248712457716,
-      "rewards/margins": 0.00116971624083817,
-      "rewards/rejected": -0.0005253913695923984,
+      "logits/chosen": -2.6865365505218506,
+      "logits/rejected": -2.6199164390563965,
+      "logps/chosen": -331.81707763671875,
+      "logps/rejected": -296.70428466796875,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 9.686091652838513e-05,
+      "rewards/margins": -0.00012469211651477963,
+      "rewards/rejected": 0.0002215529966633767,
       "step": 20
     },
     {
       "epoch": 0.06279434850863422,
-      "grad_norm": 7.732991459064897,
+      "grad_norm": 8.230332507186178,
       "learning_rate": 3.1249999999999997e-07,
-      "logits/chosen": -2.71269154548645,
-      "logits/rejected": -2.614644765853882,
-      "logps/chosen": -297.6011657714844,
-      "logps/rejected": -253.3177947998047,
-      "loss": 0.6908,
+      "logits/chosen": -2.6669394969940186,
+      "logits/rejected": -2.5895800590515137,
+      "logps/chosen": -310.1693420410156,
+      "logps/rejected": -260.9254455566406,
+      "loss": 0.6912,
       "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.005987043492496014,
-      "rewards/margins": 0.005661585368216038,
-      "rewards/rejected": 0.0003254577750340104,
+      "rewards/chosen": 0.003129849676042795,
+      "rewards/margins": 0.004864652641117573,
+      "rewards/rejected": -0.0017348021501675248,
       "step": 30
     },
     {
       "epoch": 0.0837257980115123,
-      "grad_norm": 7.653673914009054,
+      "grad_norm": 8.00937157715773,
       "learning_rate": 4.1666666666666667e-07,
-      "logits/chosen": -2.600621461868286,
-      "logits/rejected": -2.5460174083709717,
-      "logps/chosen": -279.81024169921875,
-      "logps/rejected": -266.014404296875,
-      "loss": 0.685,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.00993821956217289,
-      "rewards/margins": 0.01584107242524624,
-      "rewards/rejected": -0.005902853794395924,
+      "logits/chosen": -2.5729217529296875,
+      "logits/rejected": -2.530136823654175,
+      "logps/chosen": -285.17449951171875,
+      "logps/rejected": -272.303955078125,
+      "loss": 0.6863,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": 0.0071965730749070644,
+      "rewards/margins": 0.013435715809464455,
+      "rewards/rejected": -0.006239141337573528,
       "step": 40
     },
     {
       "epoch": 0.10465724751439037,
-      "grad_norm": 8.353830361973774,
+      "grad_norm": 8.079059688602763,
       "learning_rate": 4.999731868769026e-07,
-      "logits/chosen": -2.632459878921509,
-      "logits/rejected": -2.5336594581604004,
-      "logps/chosen": -282.0531005859375,
-      "logps/rejected": -278.79656982421875,
-      "loss": 0.6753,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": 0.01642102375626564,
-      "rewards/margins": 0.034309130162000656,
-      "rewards/rejected": -0.017888108268380165,
+      "logits/chosen": -2.601407051086426,
+      "logits/rejected": -2.5244762897491455,
+      "logps/chosen": -292.8643798828125,
+      "logps/rejected": -286.71905517578125,
+      "loss": 0.6785,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.011629783548414707,
+      "rewards/margins": 0.02448815107345581,
+      "rewards/rejected": -0.012858365662395954,
       "step": 50
     },
     {
       "epoch": 0.12558869701726844,
-      "grad_norm": 9.167056957625269,
+      "grad_norm": 9.017052709484783,
       "learning_rate": 4.990353313429303e-07,
-      "logits/chosen": -2.657794237136841,
-      "logits/rejected": -2.5765810012817383,
-      "logps/chosen": -259.9486389160156,
-      "logps/rejected": -256.295166015625,
-      "loss": 0.663,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": 0.02387804351747036,
-      "rewards/margins": 0.08073899894952774,
-      "rewards/rejected": -0.056860946118831635,
+      "logits/chosen": -2.6307859420776367,
+      "logits/rejected": -2.5665087699890137,
+      "logps/chosen": -265.0750732421875,
+      "logps/rejected": -258.7865905761719,
+      "loss": 0.6675,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": 0.021959755569696426,
+      "rewards/margins": 0.06315209716558456,
+      "rewards/rejected": -0.04119233787059784,
       "step": 60
     },
     {
       "epoch": 0.14652014652014653,
-      "grad_norm": 8.724078011965831,
+      "grad_norm": 8.264582912076714,
       "learning_rate": 4.967625656594781e-07,
-      "logits/chosen": -2.545210599899292,
-      "logits/rejected": -2.499584913253784,
-      "logps/chosen": -307.4738464355469,
-      "logps/rejected": -303.07073974609375,
-      "loss": 0.6418,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.054794955998659134,
-      "rewards/margins": 0.11737842857837677,
-      "rewards/rejected": -0.1721733808517456,
+      "logits/chosen": -2.5459885597229004,
+      "logits/rejected": -2.5088202953338623,
+      "logps/chosen": -312.3421630859375,
+      "logps/rejected": -306.2401428222656,
+      "loss": 0.6493,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.004852661397308111,
+      "rewards/margins": 0.10119061172008514,
+      "rewards/rejected": -0.1060432642698288,
       "step": 70
     },
     {
       "epoch": 0.1674515960230246,
-      "grad_norm": 11.143931467386217,
+      "grad_norm": 10.960281749614031,
       "learning_rate": 4.93167072587771e-07,
-      "logits/chosen": -2.6561365127563477,
-      "logits/rejected": -2.5057692527770996,
-      "logps/chosen": -350.4751892089844,
-      "logps/rejected": -290.5388488769531,
-      "loss": 0.6268,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.09161636233329773,
-      "rewards/margins": 0.19298198819160461,
-      "rewards/rejected": -0.28459829092025757,
+      "logits/chosen": -2.6571133136749268,
+      "logits/rejected": -2.5383903980255127,
+      "logps/chosen": -354.3619689941406,
+      "logps/rejected": -291.69244384765625,
+      "loss": 0.6364,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.05542845278978348,
+      "rewards/margins": 0.1628389060497284,
+      "rewards/rejected": -0.21826735138893127,
       "step": 80
     },
     {
       "epoch": 0.18838304552590268,
-      "grad_norm": 12.494281610138923,
+      "grad_norm": 13.098090567021211,
       "learning_rate": 4.882681251368548e-07,
-      "logits/chosen": -2.5815837383270264,
-      "logits/rejected": -2.5117011070251465,
-      "logps/chosen": -282.310791015625,
-      "logps/rejected": -313.5682067871094,
-      "loss": 0.6067,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.22465059161186218,
-      "rewards/margins": 0.21727819740772247,
-      "rewards/rejected": -0.44192880392074585,
+      "logits/chosen": -2.6158549785614014,
+      "logits/rejected": -2.5619349479675293,
+      "logps/chosen": -288.7886047363281,
+      "logps/rejected": -311.8244934082031,
+      "loss": 0.6093,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.21366631984710693,
+      "rewards/margins": 0.22560691833496094,
+      "rewards/rejected": -0.4392732083797455,
       "step": 90
     },
     {
       "epoch": 0.20931449502878074,
-      "grad_norm": 11.897397709020806,
+      "grad_norm": 10.707751058389576,
       "learning_rate": 4.820919832540181e-07,
-      "logits/chosen": -2.59562611579895,
-      "logits/rejected": -2.5282070636749268,
-      "logps/chosen": -338.21673583984375,
-      "logps/rejected": -348.1116027832031,
-      "loss": 0.6072,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.31439146399497986,
-      "rewards/margins": 0.3437823951244354,
-      "rewards/rejected": -0.6581738591194153,
+      "logits/chosen": -2.6393892765045166,
+      "logits/rejected": -2.5948047637939453,
+      "logps/chosen": -343.1591491699219,
+      "logps/rejected": -353.9322814941406,
+      "loss": 0.6065,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.2791406810283661,
+      "rewards/margins": 0.35397782921791077,
+      "rewards/rejected": -0.6331185102462769,
       "step": 100
     },
     {
       "epoch": 0.2302459445316588,
-      "grad_norm": 16.76442489393267,
+      "grad_norm": 12.745950954246485,
       "learning_rate": 4.7467175306295647e-07,
-      "logits/chosen": -2.593721866607666,
-      "logits/rejected": -2.5085806846618652,
-      "logps/chosen": -343.6995849609375,
-      "logps/rejected": -360.6001892089844,
-      "loss": 0.6009,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -0.46031326055526733,
-      "rewards/margins": 0.33665376901626587,
-      "rewards/rejected": -0.796967089176178,
+      "logits/chosen": -2.711115837097168,
+      "logits/rejected": -2.6523733139038086,
+      "logps/chosen": -335.3150634765625,
+      "logps/rejected": -345.87237548828125,
+      "loss": 0.6078,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.2736280560493469,
+      "rewards/margins": 0.2958160936832428,
+      "rewards/rejected": -0.5694441795349121,
       "step": 110
     },
     {
       "epoch": 0.25117739403453687,
-      "grad_norm": 12.98951659559394,
+      "grad_norm": 12.922005253605358,
       "learning_rate": 4.6604720940421207e-07,
-      "logits/chosen": -2.31950044631958,
-      "logits/rejected": -2.2902190685272217,
-      "logps/chosen": -332.3330993652344,
-      "logps/rejected": -360.6295166015625,
-      "loss": 0.579,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.5197553634643555,
-      "rewards/margins": 0.4039463996887207,
-      "rewards/rejected": -0.9237018823623657,
+      "logits/chosen": -2.643698215484619,
+      "logits/rejected": -2.6443088054656982,
+      "logps/chosen": -316.6920166015625,
+      "logps/rejected": -346.8565979003906,
+      "loss": 0.5867,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.32213813066482544,
+      "rewards/margins": 0.4012001156806946,
+      "rewards/rejected": -0.7233381867408752,
       "step": 120
     },
     {
       "epoch": 0.272108843537415,
-      "grad_norm": 13.545649928916575,
+      "grad_norm": 14.301862772426597,
       "learning_rate": 4.5626458262912735e-07,
-      "logits/chosen": -2.247131824493408,
-      "logits/rejected": -2.171808958053589,
-      "logps/chosen": -331.72137451171875,
-      "logps/rejected": -355.0787658691406,
-      "loss": 0.5595,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.3730407953262329,
-      "rewards/margins": 0.37541478872299194,
-      "rewards/rejected": -0.7484556436538696,
+      "logits/chosen": -2.71155047416687,
+      "logits/rejected": -2.6922905445098877,
+      "logps/chosen": -340.5058288574219,
+      "logps/rejected": -356.29541015625,
+      "loss": 0.568,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.40051087737083435,
+      "rewards/margins": 0.34178003668785095,
+      "rewards/rejected": -0.7422909140586853,
       "step": 130
     },
     {
       "epoch": 0.29304029304029305,
-      "grad_norm": 20.162985308235303,
+      "grad_norm": 19.29337794069041,
       "learning_rate": 4.453763107901675e-07,
-      "logits/chosen": -2.0727458000183105,
-      "logits/rejected": -1.917345404624939,
-      "logps/chosen": -367.02081298828125,
-      "logps/rejected": -371.6455993652344,
-      "loss": 0.5624,
+      "logits/chosen": -2.7858147621154785,
+      "logits/rejected": -2.755225419998169,
+      "logps/chosen": -368.2306213378906,
+      "logps/rejected": -366.3923645019531,
+      "loss": 0.5792,
       "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.41769176721572876,
-      "rewards/margins": 0.5499740839004517,
-      "rewards/rejected": -0.9676656723022461,
+      "rewards/chosen": -0.3208185136318207,
+      "rewards/margins": 0.485020250082016,
+      "rewards/rejected": -0.8058387637138367,
       "step": 140
     },
     {
       "epoch": 0.3139717425431711,
-      "grad_norm": 19.0565927001832,
+      "grad_norm": 20.243585876180912,
       "learning_rate": 4.3344075855595097e-07,
-      "logits/chosen": -1.912172555923462,
-      "logits/rejected": -1.765091896057129,
-      "logps/chosen": -339.38177490234375,
-      "logps/rejected": -350.7415771484375,
-      "loss": 0.5677,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.49130716919898987,
-      "rewards/margins": 0.4586152136325836,
-      "rewards/rejected": -0.9499223828315735,
+      "logits/chosen": -2.7599551677703857,
+      "logits/rejected": -2.7298169136047363,
+      "logps/chosen": -338.7522888183594,
+      "logps/rejected": -336.2152404785156,
+      "loss": 0.5849,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.4120866358280182,
+      "rewards/margins": 0.35028699040412903,
+      "rewards/rejected": -0.7623735666275024,
       "step": 150
     },
     {
       "epoch": 0.3349031920460492,
-      "grad_norm": 15.74857851317074,
+      "grad_norm": 18.324736579566704,
       "learning_rate": 4.2052190435769554e-07,
-      "logits/chosen": -1.90102219581604,
-      "logits/rejected": -1.661292314529419,
-      "logps/chosen": -325.9547424316406,
-      "logps/rejected": -354.97247314453125,
-      "loss": 0.5665,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.4339516758918762,
-      "rewards/margins": 0.566411018371582,
-      "rewards/rejected": -1.0003626346588135,
+      "logits/chosen": -2.8034064769744873,
+      "logits/rejected": -2.7623839378356934,
+      "logps/chosen": -342.80474853515625,
+      "logps/rejected": -357.0664978027344,
+      "loss": 0.5882,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.5081926584243774,
+      "rewards/margins": 0.4391111433506012,
+      "rewards/rejected": -0.9473037719726562,
       "step": 160
     },
     {
       "epoch": 0.35583464154892724,
-      "grad_norm": 22.11997978366815,
+      "grad_norm": 16.282660960565977,
       "learning_rate": 4.0668899744407567e-07,
-      "logits/chosen": -1.5486009120941162,
-      "logits/rejected": -1.4190781116485596,
-      "logps/chosen": -326.2482604980469,
-      "logps/rejected": -345.6896057128906,
-      "loss": 0.5715,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.5852349996566772,
-      "rewards/margins": 0.4395717680454254,
-      "rewards/rejected": -1.0248068571090698,
+      "logits/chosen": -2.645301103591919,
+      "logits/rejected": -2.628730058670044,
+      "logps/chosen": -332.9082946777344,
+      "logps/rejected": -339.78851318359375,
+      "loss": 0.5924,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.5716486573219299,
+      "rewards/margins": 0.3343070447444916,
+      "rewards/rejected": -0.9059556722640991,
       "step": 170
     },
     {
       "epoch": 0.37676609105180536,
-      "grad_norm": 19.615157043086096,
+      "grad_norm": 15.21087493482888,
       "learning_rate": 3.920161866827889e-07,
-      "logits/chosen": -1.4963265657424927,
-      "logits/rejected": -1.352418303489685,
-      "logps/chosen": -322.8550720214844,
-      "logps/rejected": -341.64959716796875,
-      "loss": 0.535,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.45726776123046875,
-      "rewards/margins": 0.46411681175231934,
-      "rewards/rejected": -0.9213846325874329,
+      "logits/chosen": -2.7087159156799316,
+      "logits/rejected": -2.672781467437744,
+      "logps/chosen": -327.3506774902344,
+      "logps/rejected": -333.36383056640625,
+      "loss": 0.5527,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.4208357334136963,
+      "rewards/margins": 0.3587748408317566,
+      "rewards/rejected": -0.7796105742454529,
       "step": 180
     },
     {
       "epoch": 0.3976975405546834,
-      "grad_norm": 22.15693930626595,
+      "grad_norm": 17.143640359397452,
       "learning_rate": 3.765821230985757e-07,
-      "logits/chosen": -1.2935736179351807,
-      "logits/rejected": -1.2584232091903687,
-      "logps/chosen": -317.90557861328125,
-      "logps/rejected": -357.7715759277344,
-      "loss": 0.5501,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -0.546875536441803,
-      "rewards/margins": 0.43576058745384216,
-      "rewards/rejected": -0.9826361536979675,
+      "logits/chosen": -2.6293439865112305,
+      "logits/rejected": -2.5997588634490967,
+      "logps/chosen": -319.8256530761719,
+      "logps/rejected": -352.70550537109375,
+      "loss": 0.563,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.47969502210617065,
+      "rewards/margins": 0.37083858251571655,
+      "rewards/rejected": -0.850533664226532,
       "step": 190
     },
     {
       "epoch": 0.4186289900575615,
-      "grad_norm": 28.687486673954695,
+      "grad_norm": 58.63667616094113,
       "learning_rate": 3.604695382782159e-07,
-      "logits/chosen": -1.3054436445236206,
-      "logits/rejected": -1.2638180255889893,
-      "logps/chosen": -305.0118103027344,
-      "logps/rejected": -359.88104248046875,
-      "loss": 0.5721,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.4132348895072937,
-      "rewards/margins": 0.4162468910217285,
-      "rewards/rejected": -0.8294817209243774,
+      "logits/chosen": -2.535177230834961,
+      "logits/rejected": -2.538287878036499,
+      "logps/chosen": -317.7572937011719,
+      "logps/rejected": -378.38128662109375,
+      "loss": 0.5741,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.5162296295166016,
+      "rewards/margins": 0.4109058976173401,
+      "rewards/rejected": -0.9271354675292969,
       "step": 200
     },
     {
       "epoch": 0.43956043956043955,
-      "grad_norm": 18.79057150911231,
+      "grad_norm": 22.07041734817236,
       "learning_rate": 3.4376480090239047e-07,
-      "logits/chosen": -1.5015009641647339,
-      "logits/rejected": -1.174789547920227,
-      "logps/chosen": -379.56378173828125,
-      "logps/rejected": -367.51824951171875,
-      "loss": 0.5705,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.591296374797821,
-      "rewards/margins": 0.5415098667144775,
-      "rewards/rejected": -1.1328063011169434,
+      "logits/chosen": -2.5014660358428955,
+      "logits/rejected": -2.402636766433716,
+      "logps/chosen": -374.38275146484375,
+      "logps/rejected": -362.0113830566406,
+      "loss": 0.5684,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.46330127120018005,
+      "rewards/margins": 0.5060497522354126,
+      "rewards/rejected": -0.9693509936332703,
       "step": 210
     },
     {
       "epoch": 0.4604918890633176,
-      "grad_norm": 18.737878732800926,
+      "grad_norm": 19.601082604792087,
       "learning_rate": 3.265574537815398e-07,
-      "logits/chosen": -1.2811614274978638,
-      "logits/rejected": -1.1760584115982056,
-      "logps/chosen": -330.80340576171875,
-      "logps/rejected": -373.9013366699219,
-      "loss": 0.5552,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.7049997448921204,
-      "rewards/margins": 0.5080182552337646,
-      "rewards/rejected": -1.2130179405212402,
+      "logits/chosen": -2.2485015392303467,
+      "logits/rejected": -2.2393312454223633,
+      "logps/chosen": -319.60406494140625,
+      "logps/rejected": -370.166259765625,
+      "loss": 0.562,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.6189771294593811,
+      "rewards/margins": 0.5012843608856201,
+      "rewards/rejected": -1.1202614307403564,
       "step": 220
     },
     {
       "epoch": 0.48142333856619574,
-      "grad_norm": 21.237231658325168,
+      "grad_norm": 18.201365099494677,
       "learning_rate": 3.0893973387735683e-07,
-      "logits/chosen": -1.5031620264053345,
-      "logits/rejected": -1.2624866962432861,
-      "logps/chosen": -332.4327697753906,
-      "logps/rejected": -387.6866149902344,
-      "loss": 0.5585,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.6327264904975891,
-      "rewards/margins": 0.7525407075881958,
-      "rewards/rejected": -1.3852671384811401,
+      "logits/chosen": -2.3294405937194824,
+      "logits/rejected": -2.226823568344116,
+      "logps/chosen": -347.8485412597656,
+      "logps/rejected": -399.61407470703125,
+      "loss": 0.5531,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.6986296772956848,
+      "rewards/margins": 0.7771102786064148,
+      "rewards/rejected": -1.4757399559020996,
       "step": 230
     },
     {
       "epoch": 0.5023547880690737,
-      "grad_norm": 22.484006804145768,
+      "grad_norm": 17.978204290263523,
       "learning_rate": 2.910060778827554e-07,
-      "logits/chosen": -1.5307586193084717,
-      "logits/rejected": -1.3196094036102295,
-      "logps/chosen": -347.9156799316406,
-      "logps/rejected": -378.3223571777344,
-      "loss": 0.5217,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.5113299489021301,
-      "rewards/margins": 0.5874797105789185,
-      "rewards/rejected": -1.0988094806671143,
+      "logits/chosen": -2.3042497634887695,
+      "logits/rejected": -2.198915481567383,
+      "logps/chosen": -367.82342529296875,
+      "logps/rejected": -391.17327880859375,
+      "loss": 0.5274,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.5975519418716431,
+      "rewards/margins": 0.5712900161743164,
+      "rewards/rejected": -1.1688419580459595,
       "step": 240
     },
     {
       "epoch": 0.5232862375719518,
-      "grad_norm": 21.947882128729784,
+      "grad_norm": 24.29328586959325,
       "learning_rate": 2.7285261601056697e-07,
-      "logits/chosen": -1.339247465133667,
-      "logits/rejected": -1.0327800512313843,
-      "logps/chosen": -349.29168701171875,
-      "logps/rejected": -384.4781494140625,
-      "loss": 0.5379,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.6346049308776855,
-      "rewards/margins": 0.722966194152832,
-      "rewards/rejected": -1.357571005821228,
+      "logits/chosen": -2.198782444000244,
+      "logits/rejected": -2.053729295730591,
+      "logps/chosen": -369.4089050292969,
+      "logps/rejected": -402.0597229003906,
+      "loss": 0.5322,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -0.7310666441917419,
+      "rewards/margins": 0.732733428478241,
+      "rewards/rejected": -1.4638001918792725,
       "step": 250
     },
     {
       "epoch": 0.54421768707483,
-      "grad_norm": 26.05656063347704,
+      "grad_norm": 22.871676623689975,
       "learning_rate": 2.5457665670441937e-07,
-      "logits/chosen": -1.2346374988555908,
-      "logits/rejected": -1.1367831230163574,
-      "logps/chosen": -348.9411315917969,
-      "logps/rejected": -383.9248046875,
-      "loss": 0.5433,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.7850319147109985,
-      "rewards/margins": 0.5753322839736938,
-      "rewards/rejected": -1.3603640794754028,
+      "logits/chosen": -2.0881309509277344,
+      "logits/rejected": -2.056945323944092,
+      "logps/chosen": -372.88616943359375,
+      "logps/rejected": -410.09320068359375,
+      "loss": 0.5374,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.9354459047317505,
+      "rewards/margins": 0.6017513275146484,
+      "rewards/rejected": -1.5371973514556885,
       "step": 260
     },
     {
       "epoch": 0.565149136577708,
-      "grad_norm": 16.33289220353537,
+      "grad_norm": 18.143041901320185,
       "learning_rate": 2.3627616503391812e-07,
-      "logits/chosen": -1.3926626443862915,
-      "logits/rejected": -1.2385615110397339,
-      "logps/chosen": -355.91156005859375,
-      "logps/rejected": -402.8101501464844,
-      "loss": 0.538,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.5515707731246948,
-      "rewards/margins": 0.7438338994979858,
-      "rewards/rejected": -1.2954046726226807,
+      "logits/chosen": -2.0207080841064453,
+      "logits/rejected": -1.9440813064575195,
+      "logps/chosen": -368.6573791503906,
+      "logps/rejected": -404.0098876953125,
+      "loss": 0.541,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.5764263868331909,
+      "rewards/margins": 0.6674422025680542,
+      "rewards/rejected": -1.2438685894012451,
       "step": 270
     },
     {
       "epoch": 0.5860805860805861,
-      "grad_norm": 21.47375551562495,
+      "grad_norm": 24.238102577124657,
       "learning_rate": 2.1804923757009882e-07,
-      "logits/chosen": -1.2104097604751587,
-      "logits/rejected": -1.0112183094024658,
-      "logps/chosen": -337.76531982421875,
-      "logps/rejected": -353.09912109375,
-      "loss": 0.5436,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.6104549169540405,
-      "rewards/margins": 0.5441664457321167,
-      "rewards/rejected": -1.1546214818954468,
+      "logits/chosen": -1.797975778579712,
+      "logits/rejected": -1.6956411600112915,
+      "logps/chosen": -339.45184326171875,
+      "logps/rejected": -357.5285339355469,
+      "loss": 0.5477,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.5778101086616516,
+      "rewards/margins": 0.5612425804138184,
+      "rewards/rejected": -1.1390526294708252,
       "step": 280
     },
     {
       "epoch": 0.6070120355834642,
-      "grad_norm": 20.910890345984367,
+      "grad_norm": 22.41015619268828,
       "learning_rate": 1.9999357655598891e-07,
-      "logits/chosen": -1.2669506072998047,
-      "logits/rejected": -1.1004546880722046,
-      "logps/chosen": -335.47161865234375,
-      "logps/rejected": -385.6603698730469,
-      "loss": 0.5427,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.8220598101615906,
-      "rewards/margins": 0.5405200719833374,
-      "rewards/rejected": -1.3625797033309937,
+      "logits/chosen": -1.8829982280731201,
+      "logits/rejected": -1.7629162073135376,
+      "logps/chosen": -342.6054382324219,
+      "logps/rejected": -391.98907470703125,
+      "loss": 0.5392,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.8052763938903809,
+      "rewards/margins": 0.5734516382217407,
+      "rewards/rejected": -1.3787280321121216,
       "step": 290
     },
     {
       "epoch": 0.6279434850863422,
-      "grad_norm": 18.856326831181125,
+      "grad_norm": 19.200480239691995,
       "learning_rate": 1.8220596619089573e-07,
-      "logits/chosen": -1.356403112411499,
-      "logits/rejected": -1.1351536512374878,
-      "logps/chosen": -395.8989562988281,
-      "logps/rejected": -410.227294921875,
-      "loss": 0.5164,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.6085205078125,
-      "rewards/margins": 0.6207507848739624,
-      "rewards/rejected": -1.2292712926864624,
+      "logits/chosen": -1.8666107654571533,
+      "logits/rejected": -1.7327511310577393,
+      "logps/chosen": -412.94073486328125,
+      "logps/rejected": -428.00225830078125,
+      "loss": 0.5268,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.7397323846817017,
+      "rewards/margins": 0.6389673948287964,
+      "rewards/rejected": -1.3786996603012085,
       "step": 300
     },
     {
       "epoch": 0.6488749345892203,
-      "grad_norm": 22.76686226367157,
+      "grad_norm": 23.166055643365755,
       "learning_rate": 1.647817538357072e-07,
-      "logits/chosen": -1.3988535404205322,
-      "logits/rejected": -1.2010142803192139,
-      "logps/chosen": -364.18408203125,
-      "logps/rejected": -386.00714111328125,
-      "loss": 0.5228,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -0.5230848789215088,
-      "rewards/margins": 0.784976601600647,
-      "rewards/rejected": -1.3080614805221558,
+      "logits/chosen": -1.8956129550933838,
+      "logits/rejected": -1.7741343975067139,
+      "logps/chosen": -382.96661376953125,
+      "logps/rejected": -403.57830810546875,
+      "loss": 0.5199,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.6354427933692932,
+      "rewards/margins": 0.7677633762359619,
+      "rewards/rejected": -1.4032061100006104,
       "step": 310
     },
     {
       "epoch": 0.6698063840920984,
-      "grad_norm": 18.988874912843485,
+      "grad_norm": 21.777286765570032,
       "learning_rate": 1.478143389201113e-07,
-      "logits/chosen": -1.226210117340088,
-      "logits/rejected": -0.9954258799552917,
-      "logps/chosen": -328.40216064453125,
-      "logps/rejected": -369.4967041015625,
-      "loss": 0.5151,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -0.63496994972229,
-      "rewards/margins": 0.7474745512008667,
-      "rewards/rejected": -1.3824446201324463,
+      "logits/chosen": -1.805687665939331,
+      "logits/rejected": -1.6587250232696533,
+      "logps/chosen": -335.69287109375,
+      "logps/rejected": -379.68743896484375,
+      "loss": 0.5135,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -0.6119715571403503,
+      "rewards/margins": 0.7994168996810913,
+      "rewards/rejected": -1.4113883972167969,
       "step": 320
     },
     {
       "epoch": 0.6907378335949764,
-      "grad_norm": 20.540627681342297,
+      "grad_norm": 23.11844824124375,
       "learning_rate": 1.3139467229135998e-07,
-      "logits/chosen": -1.3592547178268433,
-      "logits/rejected": -1.3184325695037842,
-      "logps/chosen": -331.37701416015625,
-      "logps/rejected": -393.223876953125,
-      "loss": 0.5226,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -0.5516895055770874,
-      "rewards/margins": 0.6636101007461548,
-      "rewards/rejected": -1.2152996063232422,
+      "logits/chosen": -1.8413807153701782,
+      "logits/rejected": -1.8144184350967407,
+      "logps/chosen": -349.6283264160156,
+      "logps/rejected": -406.96295166015625,
+      "loss": 0.5224,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.6430560350418091,
+      "rewards/margins": 0.6470705270767212,
+      "rewards/rejected": -1.2901265621185303,
       "step": 330
     },
     {
       "epoch": 0.7116692830978545,
-      "grad_norm": 25.42987653920307,
+      "grad_norm": 24.015437020386692,
       "learning_rate": 1.1561076868822755e-07,
-      "logits/chosen": -1.2324409484863281,
-      "logits/rejected": -1.0954737663269043,
-      "logps/chosen": -362.8447265625,
-      "logps/rejected": -403.8171081542969,
-      "loss": 0.5213,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.6163730621337891,
-      "rewards/margins": 0.6569727659225464,
-      "rewards/rejected": -1.273345708847046,
+      "logits/chosen": -1.717903733253479,
+      "logits/rejected": -1.6341816186904907,
+      "logps/chosen": -378.47442626953125,
+      "logps/rejected": -422.95654296875,
+      "loss": 0.5249,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.7014169692993164,
+      "rewards/margins": 0.6979398131370544,
+      "rewards/rejected": -1.3993569612503052,
       "step": 340
     },
     {
       "epoch": 0.7326007326007326,
-      "grad_norm": 20.6687240224169,
+      "grad_norm": 28.45852201826551,
       "learning_rate": 1.0054723495346482e-07,
-      "logits/chosen": -1.2725862264633179,
-      "logits/rejected": -1.0163639783859253,
-      "logps/chosen": -324.8491516113281,
-      "logps/rejected": -372.7526550292969,
-      "loss": 0.4955,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -0.5732508897781372,
-      "rewards/margins": 0.7606478929519653,
-      "rewards/rejected": -1.333898663520813,
+      "logits/chosen": -1.7926820516586304,
+      "logits/rejected": -1.6367231607437134,
+      "logps/chosen": -344.37890625,
+      "logps/rejected": -383.24432373046875,
+      "loss": 0.496,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.6705678701400757,
+      "rewards/margins": 0.6647164225578308,
+      "rewards/rejected": -1.3352842330932617,
       "step": 350
     },
     {
       "epoch": 0.7535321821036107,
-      "grad_norm": 19.600987167350965,
+      "grad_norm": 20.39869216136763,
       "learning_rate": 8.628481651367875e-08,
-      "logits/chosen": -1.1491715908050537,
-      "logits/rejected": -0.8824012875556946,
-      "logps/chosen": -378.06512451171875,
-      "logps/rejected": -407.40948486328125,
-      "loss": 0.5428,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.6164501309394836,
-      "rewards/margins": 0.7409273386001587,
-      "rewards/rejected": -1.357377529144287,
+      "logits/chosen": -1.7070884704589844,
+      "logits/rejected": -1.5379220247268677,
+      "logps/chosen": -400.3146667480469,
+      "logps/rejected": -420.22015380859375,
+      "loss": 0.549,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.7352498769760132,
+      "rewards/margins": 0.6947922706604004,
+      "rewards/rejected": -1.4300422668457031,
       "step": 360
     },
     {
       "epoch": 0.7744636316064888,
-      "grad_norm": 21.557147229171978,
+      "grad_norm": 27.00468749784755,
       "learning_rate": 7.289996455765748e-08,
-      "logits/chosen": -1.1593047380447388,
-      "logits/rejected": -0.9495538473129272,
-      "logps/chosen": -332.91485595703125,
-      "logps/rejected": -369.48150634765625,
-      "loss": 0.5286,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -0.571354329586029,
-      "rewards/margins": 0.722070038318634,
-      "rewards/rejected": -1.293424367904663,
+      "logits/chosen": -1.6615266799926758,
+      "logits/rejected": -1.5225656032562256,
+      "logps/chosen": -355.3984069824219,
+      "logps/rejected": -392.2328186035156,
+      "loss": 0.5269,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.7128881216049194,
+      "rewards/margins": 0.735795795917511,
+      "rewards/rejected": -1.4486840963363647,
       "step": 370
     },
     {
       "epoch": 0.7953950811093669,
-      "grad_norm": 22.757905962081995,
+      "grad_norm": 20.51420624974921,
       "learning_rate": 6.046442623320145e-08,
-      "logits/chosen": -0.9251530766487122,
-      "logits/rejected": -0.8378445506095886,
-      "logps/chosen": -339.1557312011719,
-      "logps/rejected": -443.68817138671875,
-      "loss": 0.5192,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.7877386212348938,
-      "rewards/margins": 0.8901177644729614,
-      "rewards/rejected": -1.6778564453125,
+      "logits/chosen": -1.4373382329940796,
+      "logits/rejected": -1.3852484226226807,
+      "logps/chosen": -353.9813537597656,
+      "logps/rejected": -454.80145263671875,
+      "loss": 0.5166,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.8685728311538696,
+      "rewards/margins": 0.8601717948913574,
+      "rewards/rejected": -1.7287447452545166,
       "step": 380
     },
     {
       "epoch": 0.8163265306122449,
-      "grad_norm": 26.244507090381283,
+      "grad_norm": 30.849969753036305,
       "learning_rate": 4.904486005914027e-08,
-      "logits/chosen": -1.2426486015319824,
-      "logits/rejected": -1.0370407104492188,
-      "logps/chosen": -425.2216796875,
-      "logps/rejected": -453.7456970214844,
-      "loss": 0.5139,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.7326547503471375,
-      "rewards/margins": 0.6568619012832642,
-      "rewards/rejected": -1.3895165920257568,
+      "logits/chosen": -1.653955101966858,
+      "logits/rejected": -1.5138956308364868,
+      "logps/chosen": -446.8741149902344,
+      "logps/rejected": -475.45977783203125,
+      "loss": 0.5155,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.8643356561660767,
+      "rewards/margins": 0.660868227481842,
+      "rewards/rejected": -1.525203824043274,
       "step": 390
     },
     {
       "epoch": 0.837257980115123,
-      "grad_norm": 24.429980563146632,
+      "grad_norm": 23.018534146860215,
       "learning_rate": 3.8702478614051345e-08,
-      "logits/chosen": -1.087192177772522,
-      "logits/rejected": -0.863726794719696,
-      "logps/chosen": -332.46685791015625,
-      "logps/rejected": -387.232421875,
-      "loss": 0.5321,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.6953538060188293,
-      "rewards/margins": 0.6994706988334656,
-      "rewards/rejected": -1.394824504852295,
+      "logits/chosen": -1.5653860569000244,
+      "logits/rejected": -1.4222080707550049,
+      "logps/chosen": -345.37451171875,
+      "logps/rejected": -400.10565185546875,
+      "loss": 0.5294,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.7452888488769531,
+      "rewards/margins": 0.7274179458618164,
+      "rewards/rejected": -1.47270667552948,
       "step": 400
     },
     {
       "epoch": 0.858189429618001,
-      "grad_norm": 21.16777347186506,
+      "grad_norm": 21.423785778797033,
       "learning_rate": 2.9492720416985e-08,
-      "logits/chosen": -1.3283381462097168,
-      "logits/rejected": -1.0876325368881226,
-      "logps/chosen": -378.5984802246094,
-      "logps/rejected": -414.51202392578125,
-      "loss": 0.531,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.6937893629074097,
-      "rewards/margins": 0.7333989143371582,
-      "rewards/rejected": -1.4271881580352783,
+      "logits/chosen": -1.7361204624176025,
+      "logits/rejected": -1.5790544748306274,
+      "logps/chosen": -392.1049499511719,
+      "logps/rejected": -428.69873046875,
+      "loss": 0.5345,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.726129412651062,
+      "rewards/margins": 0.7564193606376648,
+      "rewards/rejected": -1.4825488328933716,
       "step": 410
     },
     {
       "epoch": 0.8791208791208791,
-      "grad_norm": 26.382095604314472,
+      "grad_norm": 25.906686849547086,
       "learning_rate": 2.1464952759020856e-08,
-      "logits/chosen": -1.1406913995742798,
-      "logits/rejected": -0.9990569353103638,
-      "logps/chosen": -338.43505859375,
-      "logps/rejected": -411.83319091796875,
-      "loss": 0.5126,
+      "logits/chosen": -1.5480293035507202,
+      "logits/rejected": -1.462304711341858,
+      "logps/chosen": -354.43505859375,
+      "logps/rejected": -425.56005859375,
+      "loss": 0.5171,
       "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.7397539019584656,
-      "rewards/margins": 0.7144732475280762,
-      "rewards/rejected": -1.4542272090911865,
+      "rewards/chosen": -0.810673713684082,
+      "rewards/margins": 0.7154702544212341,
+      "rewards/rejected": -1.5261439085006714,
       "step": 420
     },
     {
       "epoch": 0.9000523286237572,
-      "grad_norm": 25.19851833189937,
+      "grad_norm": 24.715416421591268,
       "learning_rate": 1.4662207078575684e-08,
-      "logits/chosen": -1.1446809768676758,
-      "logits/rejected": -0.9151161313056946,
-      "logps/chosen": -371.24432373046875,
-      "logps/rejected": -428.16436767578125,
-      "loss": 0.5024,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -0.682637095451355,
-      "rewards/margins": 0.731964647769928,
-      "rewards/rejected": -1.4146016836166382,
+      "logits/chosen": -1.5542974472045898,
+      "logits/rejected": -1.385867714881897,
+      "logps/chosen": -381.9129943847656,
+      "logps/rejected": -429.78900146484375,
+      "loss": 0.5057,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.7492018342018127,
+      "rewards/margins": 0.7330363988876343,
+      "rewards/rejected": -1.4822382926940918,
       "step": 430
     },
     {
       "epoch": 0.9209837781266352,
-      "grad_norm": 23.916130656759286,
+      "grad_norm": 26.044110008316927,
       "learning_rate": 9.12094829893642e-09,
-      "logits/chosen": -1.3279728889465332,
-      "logits/rejected": -1.114848017692566,
-      "logps/chosen": -332.7235412597656,
-      "logps/rejected": -360.280029296875,
-      "loss": 0.5197,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.6455050706863403,
-      "rewards/margins": 0.6436306238174438,
-      "rewards/rejected": -1.2891355752944946,
+      "logits/chosen": -1.6991631984710693,
+      "logits/rejected": -1.5472667217254639,
+      "logps/chosen": -346.50555419921875,
+      "logps/rejected": -379.22064208984375,
+      "loss": 0.5129,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.692093551158905,
+      "rewards/margins": 0.6859078407287598,
+      "rewards/rejected": -1.37800133228302,
       "step": 440
     },
     {
       "epoch": 0.9419152276295133,
-      "grad_norm": 22.16715594060594,
+      "grad_norm": 20.37342934046794,
       "learning_rate": 4.8708793644441086e-09,
-      "logits/chosen": -1.0243772268295288,
-      "logits/rejected": -0.8716105222702026,
-      "logps/chosen": -358.4526672363281,
-      "logps/rejected": -418.63916015625,
-      "loss": 0.5172,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.7147374749183655,
-      "rewards/margins": 0.7633405923843384,
-      "rewards/rejected": -1.4780781269073486,
+      "logits/chosen": -1.4247163534164429,
+      "logits/rejected": -1.3255692720413208,
+      "logps/chosen": -370.42413330078125,
+      "logps/rejected": -431.6888732910156,
+      "loss": 0.5152,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.7839125394821167,
+      "rewards/margins": 0.773055374622345,
+      "rewards/rejected": -1.5569679737091064,
       "step": 450
     },
     {
       "epoch": 0.9628466771323915,
-      "grad_norm": 39.08384701109071,
+      "grad_norm": 25.986771829001402,
       "learning_rate": 1.9347820230782295e-09,
-      "logits/chosen": -1.2375072240829468,
-      "logits/rejected": -0.9246547818183899,
-      "logps/chosen": -354.1645812988281,
-      "logps/rejected": -374.7205810546875,
-      "loss": 0.5302,
+      "logits/chosen": -1.601322889328003,
+      "logits/rejected": -1.3769402503967285,
+      "logps/chosen": -367.97900390625,
+      "logps/rejected": -388.46136474609375,
+      "loss": 0.522,
       "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.7204490900039673,
-      "rewards/margins": 0.6771665811538696,
-      "rewards/rejected": -1.3976157903671265,
+      "rewards/chosen": -0.7656736969947815,
+      "rewards/margins": 0.7120579481124878,
+      "rewards/rejected": -1.4777315855026245,
       "step": 460
     },
     {
       "epoch": 0.9837781266352695,
-      "grad_norm": 21.008394137433164,
+      "grad_norm": 23.11351679142545,
       "learning_rate": 3.2839470889836627e-10,
-      "logits/chosen": -1.2062015533447266,
-      "logits/rejected": -1.0119010210037231,
-      "logps/chosen": -373.7929992675781,
-      "logps/rejected": -409.3150634765625,
-      "loss": 0.5174,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.6295822858810425,
-      "rewards/margins": 0.6495110392570496,
-      "rewards/rejected": -1.2790933847427368,
+      "logits/chosen": -1.5826936960220337,
+      "logits/rejected": -1.4511644840240479,
+      "logps/chosen": -392.7727966308594,
+      "logps/rejected": -427.092041015625,
+      "loss": 0.5147,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.6837440133094788,
+      "rewards/margins": 0.6925565600395203,
+      "rewards/rejected": -1.376300573348999,
       "step": 470
     },
     {
       "epoch": 0.9984301412872841,
-      "eval_logits/chosen": -1.0303078889846802,
-      "eval_logits/rejected": -0.8834976553916931,
-      "eval_logps/chosen": -334.1723937988281,
-      "eval_logps/rejected": -409.29547119140625,
-      "eval_loss": 0.5259878039360046,
-      "eval_rewards/accuracies": 0.77734375,
-      "eval_rewards/chosen": -0.6380884647369385,
-      "eval_rewards/margins": 0.7834274172782898,
-      "eval_rewards/rejected": -1.421515941619873,
-      "eval_runtime": 167.8072,
-      "eval_samples_per_second": 11.918,
-      "eval_steps_per_second": 0.191,
+      "eval_logits/chosen": -1.4815254211425781,
+      "eval_logits/rejected": -1.3790271282196045,
+      "eval_logps/chosen": -348.12322998046875,
+      "eval_logps/rejected": -425.112060546875,
+      "eval_loss": 0.5279496908187866,
+      "eval_rewards/accuracies": 0.78125,
+      "eval_rewards/chosen": -0.6818673610687256,
+      "eval_rewards/margins": 0.8081312775611877,
+      "eval_rewards/rejected": -1.4899988174438477,
+      "eval_runtime": 169.0575,
+      "eval_samples_per_second": 11.83,
+      "eval_steps_per_second": 0.189,
       "step": 477
     },
     {
       "epoch": 0.9984301412872841,
       "step": 477,
       "total_flos": 0.0,
-      "train_loss": 0.5633771029658288,
-      "train_runtime": 15455.4899,
-      "train_samples_per_second": 3.955,
-      "train_steps_per_second": 0.031
+      "train_loss": 0.5669088098737929,
+      "train_runtime": 15771.2037,
+      "train_samples_per_second": 3.876,
+      "train_steps_per_second": 0.03
     }
   ],
   "logging_steps": 10,