Model save

README.md

@@ -16,15 +16,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model was trained from scratch on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.4978
-- Rewards/chosen: -1.5087
-- Rewards/rejected: -2.5471
-- Rewards/accuracies: 0.7930
-- Rewards/margins: 1.0384
-- Logps/rejected: -511.8669
-- Logps/chosen: -407.1804
-- Logits/rejected: -1.0305
-- Logits/chosen: -1.1367
+- Loss: 0.5002
+- Rewards/chosen: -1.1342
+- Rewards/rejected: -2.1412
+- Rewards/accuracies: 0.7617
+- Rewards/margins: 1.0070
+- Logps/rejected: -470.2213
+- Logps/chosen: -368.5224
+- Logits/rejected: -1.8791
+- Logits/chosen: -1.9287
 
 ## Model description
 
@@ -61,7 +61,7 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.4792        | 0.9984 | 477  | 0.4978          | -1.5087        | -2.5471          | 0.7930             | 1.0384          | -511.8669      | -407.1804    | -1.0305         | -1.1367       |
+| 0.4888        | 0.9984 | 477  | 0.5002          | -1.1342        | -2.1412          | 0.7617             | 1.0070          | -470.2213      | -368.5224    | -1.8791         | -1.9287       |
 
 
 ### Framework versions

all_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.9984301412872841,
     "total_flos": 0.0,
-    "train_loss": 0.5411407262774134,
-    "train_runtime": 15368.6971,
+    "train_loss": 0.5381635759861464,
+    "train_runtime": 15576.9752,
     "train_samples": 61134,
-    "train_samples_per_second": 3.978,
+    "train_samples_per_second": 3.925,
     "train_steps_per_second": 0.031
 }

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "data/zephyr-7b-sft-lr5e-6/checkpoint-900",
+  "_name_or_path": "data/zephyr-7b-sft-lr5e-6/checkpoint-300",
   "architectures": [
     "MistralForCausalLM"
   ],

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:61337bcdda864c1debde445f237789a84bba5adb19ae06a9000c8dabbc523464
+oid sha256:28fba8223ac326195a249377fc257b8c46e9763b1ecd6cca2555ade0b0c8e07a
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7f9abb07aca47bd6ce2a6469cb9656fe74ba4a302cd9ebf1f2503b315b366309
+oid sha256:25bd6561d277329e7548c78896c57d0bddb6a49b9613e327dc1817685705d2ed
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d976abcad140f6069ee72efd1f90908984c8d470fe9d409e8d6b3994c86d5505
+oid sha256:b162ff43e267aa21d50b846ab7074628541913e9fae77bc6bfde7817e2afecce
 size 4540516344

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.9984301412872841,
     "total_flos": 0.0,
-    "train_loss": 0.5411407262774134,
-    "train_runtime": 15368.6971,
+    "train_loss": 0.5381635759861464,
+    "train_runtime": 15576.9752,
     "train_samples": 61134,
-    "train_samples_per_second": 3.978,
+    "train_samples_per_second": 3.925,
     "train_steps_per_second": 0.031
 }

trainer_state.json

@@ -10,12 +10,12 @@
   "log_history": [
     {
       "epoch": 0.0020931449502878076,
-      "grad_norm": 9.329335073969197,
+      "grad_norm": 9.116931349918216,
       "learning_rate": 1.0416666666666666e-08,
-      "logits/chosen": -2.5794811248779297,
-      "logits/rejected": -2.497187376022339,
-      "logps/chosen": -304.32061767578125,
-      "logps/rejected": -352.47076416015625,
+      "logits/chosen": -2.5019044876098633,
+      "logits/rejected": -2.413855791091919,
+      "logps/chosen": -309.30059814453125,
+      "logps/rejected": -354.3498840332031,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -25,732 +25,732 @@
     },
     {
       "epoch": 0.020931449502878074,
-      "grad_norm": 9.39839175898195,
+      "grad_norm": 9.28643570519692,
       "learning_rate": 1.0416666666666667e-07,
-      "logits/chosen": -2.3059210777282715,
-      "logits/rejected": -2.2526416778564453,
-      "logps/chosen": -295.0247802734375,
-      "logps/rejected": -266.5336608886719,
-      "loss": 0.6931,
+      "logits/chosen": -2.224431276321411,
+      "logits/rejected": -2.1674001216888428,
+      "logps/chosen": -294.14501953125,
+      "logps/rejected": -264.65106201171875,
+      "loss": 0.6932,
       "rewards/accuracies": 0.4652777910232544,
-      "rewards/chosen": 0.0005156359984539449,
-      "rewards/margins": 0.00017523310089018196,
-      "rewards/rejected": 0.00034040288301184773,
+      "rewards/chosen": 0.0006054624682292342,
+      "rewards/margins": 0.0005378619534894824,
+      "rewards/rejected": 6.76005583954975e-05,
       "step": 10
     },
     {
       "epoch": 0.04186289900575615,
-      "grad_norm": 8.636682548661312,
+      "grad_norm": 8.709334184766346,
       "learning_rate": 2.0833333333333333e-07,
-      "logits/chosen": -2.4125330448150635,
-      "logits/rejected": -2.3149588108062744,
-      "logps/chosen": -298.82635498046875,
-      "logps/rejected": -274.46929931640625,
+      "logits/chosen": -2.33512544631958,
+      "logits/rejected": -2.2312989234924316,
+      "logps/chosen": -297.8651123046875,
+      "logps/rejected": -273.5413513183594,
       "loss": 0.6919,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.005640419200062752,
-      "rewards/margins": 0.003121997695416212,
-      "rewards/rejected": 0.0025184215046465397,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.004999921191483736,
+      "rewards/margins": 0.002726882230490446,
+      "rewards/rejected": 0.0022730384953320026,
       "step": 20
     },
     {
       "epoch": 0.06279434850863422,
-      "grad_norm": 8.927065331205007,
+      "grad_norm": 8.957177981793915,
       "learning_rate": 3.1249999999999997e-07,
-      "logits/chosen": -2.392803907394409,
-      "logits/rejected": -2.2847275733947754,
-      "logps/chosen": -280.1544189453125,
-      "logps/rejected": -239.70877075195312,
-      "loss": 0.687,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": 0.03257297724485397,
-      "rewards/margins": 0.012740666046738625,
-      "rewards/rejected": 0.0198323093354702,
+      "logits/chosen": -2.3100218772888184,
+      "logits/rejected": -2.194242477416992,
+      "logps/chosen": -279.1968078613281,
+      "logps/rejected": -238.60971069335938,
+      "loss": 0.6874,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.029854198917746544,
+      "rewards/margins": 0.01208762638270855,
+      "rewards/rejected": 0.017766574397683144,
       "step": 30
     },
     {
       "epoch": 0.0837257980115123,
-      "grad_norm": 9.0799492278356,
+      "grad_norm": 9.065418527794424,
       "learning_rate": 4.1666666666666667e-07,
-      "logits/chosen": -2.3027894496917725,
-      "logits/rejected": -2.238743543624878,
-      "logps/chosen": -251.82846069335938,
-      "logps/rejected": -253.49826049804688,
-      "loss": 0.6732,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": 0.05440131947398186,
-      "rewards/margins": 0.045613545924425125,
-      "rewards/rejected": 0.008787767961621284,
+      "logits/chosen": -2.209446430206299,
+      "logits/rejected": -2.1350502967834473,
+      "logps/chosen": -251.3109130859375,
+      "logps/rejected": -252.453125,
+      "loss": 0.6741,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": 0.04785465821623802,
+      "rewards/margins": 0.042177893221378326,
+      "rewards/rejected": 0.005676767788827419,
       "step": 40
     },
     {
       "epoch": 0.10465724751439037,
-      "grad_norm": 9.67931226984001,
+      "grad_norm": 9.795340272439843,
       "learning_rate": 4.999731868769026e-07,
-      "logits/chosen": -2.3619537353515625,
-      "logits/rejected": -2.2600178718566895,
-      "logps/chosen": -268.5810852050781,
-      "logps/rejected": -272.5931396484375,
+      "logits/chosen": -2.2750747203826904,
+      "logits/rejected": -2.1647789478302,
+      "logps/chosen": -267.8486328125,
+      "logps/rejected": -271.6601867675781,
       "loss": 0.6518,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.018271328881382942,
-      "rewards/margins": 0.09364119917154312,
-      "rewards/rejected": -0.07536985725164413,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.010790064930915833,
+      "rewards/margins": 0.09177907556295395,
+      "rewards/rejected": -0.08098901063203812,
       "step": 50
     },
     {
       "epoch": 0.12558869701726844,
-      "grad_norm": 14.635644466163559,
+      "grad_norm": 23.062768320336673,
       "learning_rate": 4.990353313429303e-07,
-      "logits/chosen": -2.3826327323913574,
-      "logits/rejected": -2.299037456512451,
-      "logps/chosen": -254.78414916992188,
-      "logps/rejected": -268.08856201171875,
-      "loss": 0.6175,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -0.077847920358181,
-      "rewards/margins": 0.22102780640125275,
-      "rewards/rejected": -0.29887574911117554,
+      "logits/chosen": -2.3312735557556152,
+      "logits/rejected": -2.242985248565674,
+      "logps/chosen": -258.3775329589844,
+      "logps/rejected": -272.57452392578125,
+      "loss": 0.6181,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.1331479549407959,
+      "rewards/margins": 0.2236930876970291,
+      "rewards/rejected": -0.3568410277366638,
       "step": 60
     },
     {
       "epoch": 0.14652014652014653,
-      "grad_norm": 16.78793982187574,
+      "grad_norm": 14.577913931787089,
       "learning_rate": 4.967625656594781e-07,
-      "logits/chosen": -2.275507926940918,
-      "logits/rejected": -2.2288684844970703,
-      "logps/chosen": -318.39434814453125,
-      "logps/rejected": -335.7054138183594,
-      "loss": 0.5965,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.31896713376045227,
-      "rewards/margins": 0.2965683341026306,
-      "rewards/rejected": -0.6155354976654053,
+      "logits/chosen": -2.2307584285736084,
+      "logits/rejected": -2.1815741062164307,
+      "logps/chosen": -315.6053466796875,
+      "logps/rejected": -336.3786315917969,
+      "loss": 0.593,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.3067965507507324,
+      "rewards/margins": 0.3270111680030823,
+      "rewards/rejected": -0.6338077187538147,
       "step": 70
     },
     {
       "epoch": 0.1674515960230246,
-      "grad_norm": 24.468986262157486,
+      "grad_norm": 23.210528569148586,
       "learning_rate": 4.93167072587771e-07,
-      "logits/chosen": -2.4226534366607666,
-      "logits/rejected": -2.2795205116271973,
-      "logps/chosen": -359.22918701171875,
-      "logps/rejected": -329.4736022949219,
-      "loss": 0.5889,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.39950594305992126,
-      "rewards/margins": 0.3952278792858124,
-      "rewards/rejected": -0.7947338819503784,
+      "logits/chosen": -2.3623719215393066,
+      "logits/rejected": -2.2053894996643066,
+      "logps/chosen": -364.2427673339844,
+      "logps/rejected": -339.1261901855469,
+      "loss": 0.5853,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.46313199400901794,
+      "rewards/margins": 0.4414239823818207,
+      "rewards/rejected": -0.9045559167861938,
       "step": 80
     },
     {
       "epoch": 0.18838304552590268,
-      "grad_norm": 17.707043570035726,
+      "grad_norm": 21.641990945172875,
       "learning_rate": 4.882681251368548e-07,
-      "logits/chosen": -2.339834690093994,
-      "logits/rejected": -2.273461103439331,
-      "logps/chosen": -299.2981872558594,
-      "logps/rejected": -351.61279296875,
-      "loss": 0.5625,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.5296919345855713,
-      "rewards/margins": 0.4784451425075531,
-      "rewards/rejected": -1.0081372261047363,
+      "logits/chosen": -2.2602686882019043,
+      "logits/rejected": -2.1903841495513916,
+      "logps/chosen": -295.21783447265625,
+      "logps/rejected": -352.05517578125,
+      "loss": 0.5576,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.4987942576408386,
+      "rewards/margins": 0.5276268124580383,
+      "rewards/rejected": -1.026421070098877,
       "step": 90
     },
     {
       "epoch": 0.20931449502878074,
-      "grad_norm": 26.69565102695701,
+      "grad_norm": 21.061771812892953,
       "learning_rate": 4.820919832540181e-07,
-      "logits/chosen": -2.260993480682373,
-      "logits/rejected": -2.1810402870178223,
-      "logps/chosen": -361.8736877441406,
-      "logps/rejected": -401.89581298828125,
-      "loss": 0.5706,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.7456570863723755,
-      "rewards/margins": 0.5644229054450989,
-      "rewards/rejected": -1.3100799322128296,
+      "logits/chosen": -2.236736536026001,
+      "logits/rejected": -2.155395984649658,
+      "logps/chosen": -344.68670654296875,
+      "logps/rejected": -387.8133544921875,
+      "loss": 0.5677,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.5947206616401672,
+      "rewards/margins": 0.5838772058486938,
+      "rewards/rejected": -1.1785976886749268,
       "step": 100
     },
     {
       "epoch": 0.2302459445316588,
-      "grad_norm": 20.282860275517262,
+      "grad_norm": 23.54686357075434,
       "learning_rate": 4.7467175306295647e-07,
-      "logits/chosen": -2.2695631980895996,
-      "logits/rejected": -2.1812210083007812,
-      "logps/chosen": -341.11517333984375,
-      "logps/rejected": -369.3460998535156,
-      "loss": 0.5746,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.560735821723938,
-      "rewards/margins": 0.4470297694206238,
-      "rewards/rejected": -1.007765531539917,
+      "logits/chosen": -2.2766032218933105,
+      "logits/rejected": -2.1909542083740234,
+      "logps/chosen": -333.171142578125,
+      "logps/rejected": -361.71038818359375,
+      "loss": 0.5712,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.4968598783016205,
+      "rewards/margins": 0.4502771496772766,
+      "rewards/rejected": -0.9471370577812195,
       "step": 110
     },
     {
       "epoch": 0.25117739403453687,
-      "grad_norm": 22.21147816496728,
+      "grad_norm": 19.869085451296872,
       "learning_rate": 4.6604720940421207e-07,
-      "logits/chosen": -2.027794361114502,
-      "logits/rejected": -2.003518581390381,
-      "logps/chosen": -317.1827392578125,
-      "logps/rejected": -364.8865051269531,
-      "loss": 0.5635,
+      "logits/chosen": -2.144845485687256,
+      "logits/rejected": -2.130959987640381,
+      "logps/chosen": -307.67059326171875,
+      "logps/rejected": -351.79937744140625,
+      "loss": 0.5614,
       "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.5613419413566589,
-      "rewards/margins": 0.5403085947036743,
-      "rewards/rejected": -1.1016504764556885,
+      "rewards/chosen": -0.47877049446105957,
+      "rewards/margins": 0.5036526918411255,
+      "rewards/rejected": -0.9824231863021851,
       "step": 120
     },
     {
       "epoch": 0.272108843537415,
-      "grad_norm": 17.405962778805314,
+      "grad_norm": 17.51341739045148,
       "learning_rate": 4.5626458262912735e-07,
-      "logits/chosen": -2.1424918174743652,
-      "logits/rejected": -2.093109607696533,
-      "logps/chosen": -340.9356994628906,
-      "logps/rejected": -373.436767578125,
-      "loss": 0.5392,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.6785966753959656,
-      "rewards/margins": 0.4390381872653961,
-      "rewards/rejected": -1.1176350116729736,
+      "logits/chosen": -2.1646664142608643,
+      "logits/rejected": -2.1193132400512695,
+      "logps/chosen": -339.0682678222656,
+      "logps/rejected": -379.1031188964844,
+      "loss": 0.5353,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.6744415760040283,
+      "rewards/margins": 0.506578803062439,
+      "rewards/rejected": -1.1810203790664673,
       "step": 130
     },
     {
       "epoch": 0.29304029304029305,
-      "grad_norm": 28.31004091852717,
+      "grad_norm": 24.41359219908897,
       "learning_rate": 4.453763107901675e-07,
-      "logits/chosen": -2.3126606941223145,
-      "logits/rejected": -2.255511999130249,
-      "logps/chosen": -362.1305236816406,
-      "logps/rejected": -381.7144775390625,
-      "loss": 0.5553,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -0.548693060874939,
-      "rewards/margins": 0.6247068047523499,
-      "rewards/rejected": -1.1733999252319336,
+      "logits/chosen": -2.1476926803588867,
+      "logits/rejected": -2.068664789199829,
+      "logps/chosen": -356.5169982910156,
+      "logps/rejected": -384.5221862792969,
+      "loss": 0.5516,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.5066072940826416,
+      "rewards/margins": 0.6951266527175903,
+      "rewards/rejected": -1.201733946800232,
       "step": 140
     },
     {
       "epoch": 0.3139717425431711,
-      "grad_norm": 24.3119082259932,
+      "grad_norm": 27.42825247197358,
       "learning_rate": 4.3344075855595097e-07,
-      "logits/chosen": -2.302546262741089,
-      "logits/rejected": -2.252924680709839,
-      "logps/chosen": -329.14752197265625,
-      "logps/rejected": -341.7516784667969,
-      "loss": 0.5638,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.5654751062393188,
-      "rewards/margins": 0.43807610869407654,
-      "rewards/rejected": -1.0035512447357178,
+      "logits/chosen": -2.1487209796905518,
+      "logits/rejected": -2.081498384475708,
+      "logps/chosen": -332.41900634765625,
+      "logps/rejected": -348.5780029296875,
+      "loss": 0.5496,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.6090599298477173,
+      "rewards/margins": 0.47456836700439453,
+      "rewards/rejected": -1.0836282968521118,
       "step": 150
     },
     {
       "epoch": 0.3349031920460492,
-      "grad_norm": 24.820352307658577,
+      "grad_norm": 23.584034728880518,
       "learning_rate": 4.2052190435769554e-07,
-      "logits/chosen": -2.3529915809631348,
-      "logits/rejected": -2.2801713943481445,
-      "logps/chosen": -323.8702697753906,
-      "logps/rejected": -359.6528625488281,
-      "loss": 0.5614,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.550986647605896,
-      "rewards/margins": 0.5704027414321899,
-      "rewards/rejected": -1.121389389038086,
+      "logits/chosen": -2.2258999347686768,
+      "logits/rejected": -2.143951177597046,
+      "logps/chosen": -362.18011474609375,
+      "logps/rejected": -409.2604064941406,
+      "loss": 0.5541,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.9483500719070435,
+      "rewards/margins": 0.686519980430603,
+      "rewards/rejected": -1.634869933128357,
       "step": 160
     },
     {
       "epoch": 0.35583464154892724,
-      "grad_norm": 25.696925524840598,
+      "grad_norm": 26.580307699089687,
       "learning_rate": 4.0668899744407567e-07,
-      "logits/chosen": -2.182185411453247,
-      "logits/rejected": -2.152573347091675,
-      "logps/chosen": -338.01385498046875,
-      "logps/rejected": -373.7718200683594,
-      "loss": 0.541,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.8493636846542358,
-      "rewards/margins": 0.5777457356452942,
-      "rewards/rejected": -1.4271094799041748,
+      "logits/chosen": -2.109285354614258,
+      "logits/rejected": -2.080744743347168,
+      "logps/chosen": -343.9925842285156,
+      "logps/rejected": -381.8763427734375,
+      "loss": 0.5386,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.9248130917549133,
+      "rewards/margins": 0.5975202322006226,
+      "rewards/rejected": -1.5223333835601807,
       "step": 170
     },
     {
       "epoch": 0.37676609105180536,
-      "grad_norm": 21.89194403141913,
+      "grad_norm": 23.319825415805855,
       "learning_rate": 3.920161866827889e-07,
-      "logits/chosen": -2.2576639652252197,
-      "logits/rejected": -2.21673846244812,
-      "logps/chosen": -374.31195068359375,
-      "logps/rejected": -404.2490234375,
-      "loss": 0.5097,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -1.093261957168579,
-      "rewards/margins": 0.5546390414237976,
-      "rewards/rejected": -1.6479008197784424,
+      "logits/chosen": -2.1809253692626953,
+      "logits/rejected": -2.138430595397949,
+      "logps/chosen": -354.91748046875,
+      "logps/rejected": -390.643798828125,
+      "loss": 0.5073,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.9161571264266968,
+      "rewards/margins": 0.609030544757843,
+      "rewards/rejected": -1.525187611579895,
       "step": 180
     },
     {
       "epoch": 0.3976975405546834,
-      "grad_norm": 22.896643777131406,
+      "grad_norm": 21.96031761304346,
       "learning_rate": 3.765821230985757e-07,
-      "logits/chosen": -2.1997172832489014,
-      "logits/rejected": -2.165952205657959,
-      "logps/chosen": -347.4014587402344,
-      "logps/rejected": -393.66583251953125,
-      "loss": 0.5438,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.9438008069992065,
-      "rewards/margins": 0.5176998972892761,
-      "rewards/rejected": -1.4615007638931274,
+      "logits/chosen": -2.0684847831726074,
+      "logits/rejected": -2.0338962078094482,
+      "logps/chosen": -348.37969970703125,
+      "logps/rejected": -403.85198974609375,
+      "loss": 0.5408,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.9659508466720581,
+      "rewards/margins": 0.6102802753448486,
+      "rewards/rejected": -1.5762312412261963,
       "step": 190
     },
     {
       "epoch": 0.4186289900575615,
-      "grad_norm": 49.114162407573595,
+      "grad_norm": 50.060138394990204,
       "learning_rate": 3.604695382782159e-07,
-      "logits/chosen": -2.129727602005005,
-      "logits/rejected": -2.128661632537842,
-      "logps/chosen": -342.37164306640625,
-      "logps/rejected": -416.89434814453125,
-      "loss": 0.5509,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.9995099902153015,
-      "rewards/margins": 0.5305387377738953,
-      "rewards/rejected": -1.5300486087799072,
+      "logits/chosen": -2.107419490814209,
+      "logits/rejected": -2.1084322929382324,
+      "logps/chosen": -313.70758056640625,
+      "logps/rejected": -391.5169982910156,
+      "loss": 0.5387,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.7262401580810547,
+      "rewards/margins": 0.5640349388122559,
+      "rewards/rejected": -1.290274977684021,
       "step": 200
     },
     {
       "epoch": 0.43956043956043955,
-      "grad_norm": 25.978762356187595,
+      "grad_norm": 23.415475415609887,
       "learning_rate": 3.4376480090239047e-07,
-      "logits/chosen": -2.210256576538086,
-      "logits/rejected": -2.120772361755371,
-      "logps/chosen": -389.1651611328125,
-      "logps/rejected": -398.2503967285156,
-      "loss": 0.5415,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.8992250561714172,
-      "rewards/margins": 0.6293545961380005,
-      "rewards/rejected": -1.528579592704773,
+      "logits/chosen": -2.1558845043182373,
+      "logits/rejected": -2.0679712295532227,
+      "logps/chosen": -382.5361022949219,
+      "logps/rejected": -394.1541748046875,
+      "loss": 0.5332,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.8439299464225769,
+      "rewards/margins": 0.6521940231323242,
+      "rewards/rejected": -1.4961239099502563,
       "step": 210
     },
     {
       "epoch": 0.4604918890633176,
-      "grad_norm": 33.51953390918859,
+      "grad_norm": 34.135780798867444,
       "learning_rate": 3.265574537815398e-07,
-      "logits/chosen": -2.1229686737060547,
-      "logits/rejected": -2.1305766105651855,
-      "logps/chosen": -319.25830078125,
-      "logps/rejected": -388.66864013671875,
-      "loss": 0.5314,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.8613080978393555,
-      "rewards/margins": 0.6536533236503601,
-      "rewards/rejected": -1.5149614810943604,
+      "logits/chosen": -1.9848287105560303,
+      "logits/rejected": -1.9900325536727905,
+      "logps/chosen": -326.7507019042969,
+      "logps/rejected": -404.930908203125,
+      "loss": 0.5201,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.9520002603530884,
+      "rewards/margins": 0.7356088757514954,
+      "rewards/rejected": -1.687609076499939,
       "step": 220
     },
     {
       "epoch": 0.48142333856619574,
-      "grad_norm": 29.955417191653368,
+      "grad_norm": 28.527689860068232,
       "learning_rate": 3.0893973387735683e-07,
-      "logits/chosen": -2.220747709274292,
-      "logits/rejected": -2.156855344772339,
-      "logps/chosen": -368.21832275390625,
-      "logps/rejected": -436.96978759765625,
-      "loss": 0.5422,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -1.153348684310913,
-      "rewards/margins": 0.8645240068435669,
-      "rewards/rejected": -2.0178725719451904,
+      "logits/chosen": -2.0887725353240967,
+      "logits/rejected": -2.014695644378662,
+      "logps/chosen": -349.72967529296875,
+      "logps/rejected": -429.88543701171875,
+      "loss": 0.5368,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.9729328155517578,
+      "rewards/margins": 0.9894140362739563,
+      "rewards/rejected": -1.9623466730117798,
       "step": 230
     },
     {
       "epoch": 0.5023547880690737,
-      "grad_norm": 27.42193643915442,
+      "grad_norm": 22.883333627448373,
       "learning_rate": 2.910060778827554e-07,
-      "logits/chosen": -2.266777992248535,
-      "logits/rejected": -2.204097270965576,
-      "logps/chosen": -393.9754638671875,
-      "logps/rejected": -437.9512634277344,
-      "loss": 0.5111,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -1.1324727535247803,
-      "rewards/margins": 0.7075997591018677,
-      "rewards/rejected": -1.8400723934173584,
+      "logits/chosen": -2.109846830368042,
+      "logits/rejected": -2.0375282764434814,
+      "logps/chosen": -386.3774108886719,
+      "logps/rejected": -440.4864196777344,
+      "loss": 0.4968,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.073874831199646,
+      "rewards/margins": 0.8030624389648438,
+      "rewards/rejected": -1.8769372701644897,
       "step": 240
     },
     {
       "epoch": 0.5232862375719518,
-      "grad_norm": 28.011124239401976,
+      "grad_norm": 25.312851481383237,
       "learning_rate": 2.7285261601056697e-07,
-      "logits/chosen": -2.204603433609009,
-      "logits/rejected": -2.123680591583252,
-      "logps/chosen": -374.8927917480469,
-      "logps/rejected": -419.709228515625,
-      "loss": 0.5194,
+      "logits/chosen": -2.0123534202575684,
+      "logits/rejected": -1.9133853912353516,
+      "logps/chosen": -386.14208984375,
+      "logps/rejected": -443.0663146972656,
+      "loss": 0.4996,
       "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -1.042311191558838,
-      "rewards/margins": 0.8302870988845825,
-      "rewards/rejected": -1.8725982904434204,
+      "rewards/chosen": -1.1668659448623657,
+      "rewards/margins": 0.9512737393379211,
+      "rewards/rejected": -2.1181397438049316,
       "step": 250
     },
     {
       "epoch": 0.54421768707483,
-      "grad_norm": 29.87367803270309,
+      "grad_norm": 34.094416716467194,
       "learning_rate": 2.5457665670441937e-07,
-      "logits/chosen": -2.066101551055908,
-      "logits/rejected": -2.0572917461395264,
-      "logps/chosen": -380.6065368652344,
-      "logps/rejected": -434.6231384277344,
-      "loss": 0.5178,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -1.2320181131362915,
-      "rewards/margins": 0.7053101062774658,
-      "rewards/rejected": -1.9373281002044678,
+      "logits/chosen": -2.0469281673431396,
+      "logits/rejected": -2.0387165546417236,
+      "logps/chosen": -352.0227355957031,
+      "logps/rejected": -415.1515197753906,
+      "loss": 0.5012,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.961257815361023,
+      "rewards/margins": 0.7900217771530151,
+      "rewards/rejected": -1.7512794733047485,
       "step": 260
     },
     {
       "epoch": 0.565149136577708,
-      "grad_norm": 25.971591354201273,
+      "grad_norm": 28.863784386605268,
       "learning_rate": 2.3627616503391812e-07,
-      "logits/chosen": -2.004692792892456,
-      "logits/rejected": -1.9637737274169922,
-      "logps/chosen": -402.45672607421875,
-      "logps/rejected": -466.58935546875,
-      "loss": 0.5224,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.1786950826644897,
-      "rewards/margins": 0.8734442591667175,
-      "rewards/rejected": -2.0521395206451416,
+      "logits/chosen": -2.0562832355499268,
+      "logits/rejected": -2.0191993713378906,
+      "logps/chosen": -379.99249267578125,
+      "logps/rejected": -446.9405212402344,
+      "loss": 0.517,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.9665490984916687,
+      "rewards/margins": 0.9049601554870605,
+      "rewards/rejected": -1.871509313583374,
       "step": 270
     },
     {
       "epoch": 0.5860805860805861,
-      "grad_norm": 32.81380916421019,
+      "grad_norm": 30.461985745500595,
       "learning_rate": 2.1804923757009882e-07,
-      "logits/chosen": -1.8292579650878906,
-      "logits/rejected": -1.7800662517547607,
-      "logps/chosen": -368.78173828125,
-      "logps/rejected": -411.6261291503906,
-      "loss": 0.5283,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -1.1611419916152954,
-      "rewards/margins": 0.676916241645813,
-      "rewards/rejected": -1.8380582332611084,
+      "logits/chosen": -1.953189492225647,
+      "logits/rejected": -1.9233779907226562,
+      "logps/chosen": -352.65679931640625,
+      "logps/rejected": -398.24676513671875,
+      "loss": 0.521,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.012357473373413,
+      "rewards/margins": 0.7160497903823853,
+      "rewards/rejected": -1.7284071445465088,
       "step": 280
     },
     {
       "epoch": 0.6070120355834642,
-      "grad_norm": 24.629742956815843,
+      "grad_norm": 26.431629125355798,
       "learning_rate": 1.9999357655598891e-07,
-      "logits/chosen": -1.9304834604263306,
-      "logits/rejected": -1.8354400396347046,
-      "logps/chosen": -361.0514221191406,
-      "logps/rejected": -428.248291015625,
-      "loss": 0.5242,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -1.1889616250991821,
-      "rewards/margins": 0.7383087277412415,
-      "rewards/rejected": -1.927270531654358,
+      "logits/chosen": -2.0837626457214355,
+      "logits/rejected": -1.9983190298080444,
+      "logps/chosen": -349.8140563964844,
+      "logps/rejected": -423.3451232910156,
+      "loss": 0.5092,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -1.0958900451660156,
+      "rewards/margins": 0.8027156591415405,
+      "rewards/rejected": -1.8986055850982666,
       "step": 290
     },
     {
       "epoch": 0.6279434850863422,
-      "grad_norm": 24.75612827312869,
+      "grad_norm": 29.400924390830824,
       "learning_rate": 1.8220596619089573e-07,
-      "logits/chosen": -1.8411821126937866,
-      "logits/rejected": -1.7451651096343994,
-      "logps/chosen": -426.62127685546875,
-      "logps/rejected": -458.081298828125,
-      "loss": 0.4965,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -1.1761503219604492,
-      "rewards/margins": 0.7339165210723877,
-      "rewards/rejected": -1.9100669622421265,
+      "logits/chosen": -2.0038347244262695,
+      "logits/rejected": -1.9328056573867798,
+      "logps/chosen": -425.2359313964844,
+      "logps/rejected": -469.41302490234375,
+      "loss": 0.4888,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.1751129627227783,
+      "rewards/margins": 0.8579627275466919,
+      "rewards/rejected": -2.0330758094787598,
       "step": 300
     },
     {
       "epoch": 0.6488749345892203,
-      "grad_norm": 31.593110395158458,
+      "grad_norm": 33.812079527501474,
       "learning_rate": 1.647817538357072e-07,
-      "logits/chosen": -1.7611089944839478,
-      "logits/rejected": -1.6688464879989624,
-      "logps/chosen": -427.68865966796875,
-      "logps/rejected": -475.7120666503906,
-      "loss": 0.4997,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -1.3469114303588867,
-      "rewards/margins": 0.9681358337402344,
-      "rewards/rejected": -2.315047264099121,
+      "logits/chosen": -2.050591230392456,
+      "logits/rejected": -1.9925072193145752,
+      "logps/chosen": -408.6422424316406,
+      "logps/rejected": -458.92236328125,
+      "loss": 0.5028,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -1.1696922779083252,
+      "rewards/margins": 0.9961409568786621,
+      "rewards/rejected": -2.1658332347869873,
       "step": 310
     },
     {
       "epoch": 0.6698063840920984,
-      "grad_norm": 30.99355606482385,
+      "grad_norm": 37.08864550510121,
       "learning_rate": 1.478143389201113e-07,
-      "logits/chosen": -1.6458816528320312,
-      "logits/rejected": -1.5158228874206543,
-      "logps/chosen": -387.9081115722656,
-      "logps/rejected": -454.47454833984375,
-      "loss": 0.4849,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -1.348028302192688,
-      "rewards/margins": 0.9673466682434082,
-      "rewards/rejected": -2.3153748512268066,
+      "logits/chosen": -2.0523581504821777,
+      "logits/rejected": -1.9676830768585205,
+      "logps/chosen": -362.476806640625,
+      "logps/rejected": -435.603515625,
+      "loss": 0.4841,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -1.1036216020584106,
+      "rewards/margins": 1.034621000289917,
+      "rewards/rejected": -2.138242721557617,
       "step": 320
     },
     {
       "epoch": 0.6907378335949764,
-      "grad_norm": 30.37220826367352,
+      "grad_norm": 25.850941149616308,
       "learning_rate": 1.3139467229135998e-07,
-      "logits/chosen": -1.711145043373108,
-      "logits/rejected": -1.6804851293563843,
-      "logps/chosen": -377.6120910644531,
-      "logps/rejected": -460.2926330566406,
-      "loss": 0.5001,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.1556519269943237,
-      "rewards/margins": 0.8555381894111633,
-      "rewards/rejected": -2.011190176010132,
+      "logits/chosen": -2.0628750324249268,
+      "logits/rejected": -2.0358595848083496,
+      "logps/chosen": -365.1703796386719,
+      "logps/rejected": -441.9532165527344,
+      "loss": 0.5018,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.0403659343719482,
+      "rewards/margins": 0.7965776324272156,
+      "rewards/rejected": -1.8369436264038086,
       "step": 330
     },
     {
       "epoch": 0.7116692830978545,
-      "grad_norm": 38.1460846269603,
+      "grad_norm": 34.62687375967783,
       "learning_rate": 1.1561076868822755e-07,
-      "logits/chosen": -1.5931702852249146,
-      "logits/rejected": -1.5212587118148804,
-      "logps/chosen": -399.8424072265625,
-      "logps/rejected": -472.65155029296875,
-      "loss": 0.4972,
+      "logits/chosen": -1.9639819860458374,
+      "logits/rejected": -1.9200565814971924,
+      "logps/chosen": -382.9055480957031,
+      "logps/rejected": -451.21844482421875,
+      "loss": 0.4997,
       "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.1858758926391602,
-      "rewards/margins": 0.9419769048690796,
-      "rewards/rejected": -2.1278529167175293,
+      "rewards/chosen": -1.0268144607543945,
+      "rewards/margins": 0.899573802947998,
+      "rewards/rejected": -1.9263885021209717,
       "step": 340
     },
     {
       "epoch": 0.7326007326007326,
-      "grad_norm": 30.619198075631438,
+      "grad_norm": 28.821747585414418,
       "learning_rate": 1.0054723495346482e-07,
-      "logits/chosen": -1.6008256673812866,
-      "logits/rejected": -1.469053030014038,
-      "logps/chosen": -377.48199462890625,
-      "logps/rejected": -447.2705078125,
-      "loss": 0.4667,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -1.2408863306045532,
-      "rewards/margins": 0.9591889381408691,
-      "rewards/rejected": -2.200075626373291,
+      "logits/chosen": -2.0305135250091553,
+      "logits/rejected": -1.9576504230499268,
+      "logps/chosen": -360.287841796875,
+      "logps/rejected": -423.4408264160156,
+      "loss": 0.4664,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -1.0807859897613525,
+      "rewards/margins": 0.892835795879364,
+      "rewards/rejected": -1.9736217260360718,
       "step": 350
     },
     {
       "epoch": 0.7535321821036107,
-      "grad_norm": 26.5580883938646,
+      "grad_norm": 25.467880760482146,
       "learning_rate": 8.628481651367875e-08,
-      "logits/chosen": -1.5030509233474731,
-      "logits/rejected": -1.3449413776397705,
-      "logps/chosen": -438.1309509277344,
-      "logps/rejected": -495.4224548339844,
-      "loss": 0.5207,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -1.379766821861267,
-      "rewards/margins": 0.9859799146652222,
-      "rewards/rejected": -2.3657467365264893,
+      "logits/chosen": -2.0199437141418457,
+      "logits/rejected": -1.9278405904769897,
+      "logps/chosen": -417.2381286621094,
+      "logps/rejected": -475.42413330078125,
+      "loss": 0.5101,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.1815179586410522,
+      "rewards/margins": 0.9967412948608398,
+      "rewards/rejected": -2.1782593727111816,
       "step": 360
     },
     {
       "epoch": 0.7744636316064888,
-      "grad_norm": 31.60257281481512,
+      "grad_norm": 26.452312032132216,
       "learning_rate": 7.289996455765748e-08,
-      "logits/chosen": -1.4332393407821655,
-      "logits/rejected": -1.2968789339065552,
-      "logps/chosen": -396.47808837890625,
-      "logps/rejected": -458.3636779785156,
-      "loss": 0.5025,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -1.3559287786483765,
-      "rewards/margins": 0.9056638479232788,
-      "rewards/rejected": -2.261592388153076,
+      "logits/chosen": -2.040602445602417,
+      "logits/rejected": -1.9673640727996826,
+      "logps/chosen": -371.0767822265625,
+      "logps/rejected": -433.1690979003906,
+      "loss": 0.5048,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.1163009405136108,
+      "rewards/margins": 0.9148439168930054,
+      "rewards/rejected": -2.0311450958251953,
       "step": 370
     },
     {
       "epoch": 0.7953950811093669,
-      "grad_norm": 36.22125453525851,
+      "grad_norm": 26.092007070398452,
       "learning_rate": 6.046442623320145e-08,
-      "logits/chosen": -1.1701750755310059,
-      "logits/rejected": -1.1165691614151,
-      "logps/chosen": -402.2992248535156,
-      "logps/rejected": -530.8674926757812,
-      "loss": 0.5,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -1.6193259954452515,
-      "rewards/margins": 1.1059458255767822,
-      "rewards/rejected": -2.725271701812744,
+      "logits/chosen": -1.8879483938217163,
+      "logits/rejected": -1.865674614906311,
+      "logps/chosen": -371.80023193359375,
+      "logps/rejected": -499.50958251953125,
+      "loss": 0.4984,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.3235037326812744,
+      "rewards/margins": 1.1101723909378052,
+      "rewards/rejected": -2.433676242828369,
       "step": 380
     },
     {
       "epoch": 0.8163265306122449,
-      "grad_norm": 36.10097946451488,
+      "grad_norm": 28.77657253530194,
       "learning_rate": 4.904486005914027e-08,
-      "logits/chosen": -1.3514797687530518,
-      "logits/rejected": -1.212864637374878,
-      "logps/chosen": -490.6700134277344,
-      "logps/rejected": -539.9706420898438,
-      "loss": 0.4882,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -1.5991289615631104,
-      "rewards/margins": 0.8608202934265137,
-      "rewards/rejected": -2.459949016571045,
+      "logits/chosen": -1.9853847026824951,
+      "logits/rejected": -1.911975622177124,
+      "logps/chosen": -458.306884765625,
+      "logps/rejected": -503.86407470703125,
+      "loss": 0.4926,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.2911863327026367,
+      "rewards/margins": 0.8143588304519653,
+      "rewards/rejected": -2.1055452823638916,
       "step": 390
     },
     {
       "epoch": 0.837257980115123,
-      "grad_norm": 28.11172601859201,
+      "grad_norm": 30.856344977247016,
       "learning_rate": 3.8702478614051345e-08,
-      "logits/chosen": -1.2067358493804932,
-      "logits/rejected": -1.0635684728622437,
-      "logps/chosen": -404.2466735839844,
-      "logps/rejected": -487.6395568847656,
-      "loss": 0.4991,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -1.5453364849090576,
-      "rewards/margins": 0.9591501355171204,
-      "rewards/rejected": -2.504486560821533,
+      "logits/chosen": -1.928967833518982,
+      "logits/rejected": -1.8698561191558838,
+      "logps/chosen": -364.50079345703125,
+      "logps/rejected": -439.862060546875,
+      "loss": 0.5065,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.1625478267669678,
+      "rewards/margins": 0.8844507336616516,
+      "rewards/rejected": -2.0469985008239746,
       "step": 400
     },
     {
       "epoch": 0.858189429618001,
-      "grad_norm": 26.352145349738,
+      "grad_norm": 30.36982208751961,
       "learning_rate": 2.9492720416985e-08,
-      "logits/chosen": -1.371030569076538,
-      "logits/rejected": -1.210618257522583,
-      "logps/chosen": -448.0904235839844,
-      "logps/rejected": -516.2791748046875,
-      "loss": 0.5036,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.5363811254501343,
-      "rewards/margins": 1.0023877620697021,
-      "rewards/rejected": -2.538768768310547,
+      "logits/chosen": -2.0435752868652344,
+      "logits/rejected": -1.9785324335098267,
+      "logps/chosen": -410.001708984375,
+      "logps/rejected": -473.53021240234375,
+      "loss": 0.5069,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.1594537496566772,
+      "rewards/margins": 0.9617475271224976,
+      "rewards/rejected": -2.121201276779175,
       "step": 410
     },
     {
       "epoch": 0.8791208791208791,
-      "grad_norm": 29.613783320470674,
+      "grad_norm": 26.44267601096535,
       "learning_rate": 2.1464952759020856e-08,
-      "logits/chosen": -1.1845066547393799,
-      "logits/rejected": -1.0983798503875732,
-      "logps/chosen": -412.7200622558594,
-      "logps/rejected": -516.0087890625,
-      "loss": 0.4935,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.6220592260360718,
-      "rewards/margins": 0.9952875971794128,
-      "rewards/rejected": -2.617347002029419,
+      "logits/chosen": -1.8928883075714111,
+      "logits/rejected": -1.8674418926239014,
+      "logps/chosen": -373.64337158203125,
+      "logps/rejected": -477.41925048828125,
+      "loss": 0.4882,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.2476003170013428,
+      "rewards/margins": 0.9974838495254517,
+      "rewards/rejected": -2.245084047317505,
       "step": 420
     },
     {
       "epoch": 0.9000523286237572,
-      "grad_norm": 27.92659999063579,
+      "grad_norm": 35.343680048124824,
       "learning_rate": 1.4662207078575684e-08,
-      "logits/chosen": -1.19228196144104,
-      "logits/rejected": -1.0212948322296143,
-      "logps/chosen": -443.21728515625,
-      "logps/rejected": -520.3425903320312,
-      "loss": 0.479,
+      "logits/chosen": -1.9129575490951538,
+      "logits/rejected": -1.8129222393035889,
+      "logps/chosen": -403.3264465332031,
+      "logps/rejected": -476.1175842285156,
+      "loss": 0.48,
       "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -1.6021983623504639,
-      "rewards/margins": 1.0000523328781128,
-      "rewards/rejected": -2.602250576019287,
+      "rewards/chosen": -1.2193301916122437,
+      "rewards/margins": 0.9621838331222534,
+      "rewards/rejected": -2.181514024734497,
       "step": 430
     },
     {
       "epoch": 0.9209837781266352,
-      "grad_norm": 29.526447811458354,
+      "grad_norm": 29.218847379942574,
       "learning_rate": 9.12094829893642e-09,
-      "logits/chosen": -1.3441697359085083,
-      "logits/rejected": -1.1822043657302856,
-      "logps/chosen": -409.3326416015625,
-      "logps/rejected": -466.71160888671875,
-      "loss": 0.4941,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.5222885608673096,
-      "rewards/margins": 0.938208281993866,
-      "rewards/rejected": -2.4604969024658203,
+      "logits/chosen": -2.0264434814453125,
+      "logits/rejected": -1.9356876611709595,
+      "logps/chosen": -368.6754455566406,
+      "logps/rejected": -425.6283264160156,
+      "loss": 0.4911,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.131156086921692,
+      "rewards/margins": 0.9324997067451477,
+      "rewards/rejected": -2.0636560916900635,
       "step": 440
     },
     {
       "epoch": 0.9419152276295133,
-      "grad_norm": 27.94896493250909,
+      "grad_norm": 28.983100441204417,
       "learning_rate": 4.8708793644441086e-09,
-      "logits/chosen": -1.0641485452651978,
-      "logits/rejected": -0.9622459411621094,
-      "logps/chosen": -422.19793701171875,
-      "logps/rejected": -521.9647216796875,
-      "loss": 0.4844,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -1.5626780986785889,
-      "rewards/margins": 1.1041589975357056,
-      "rewards/rejected": -2.666837215423584,
+      "logits/chosen": -1.76410710811615,
+      "logits/rejected": -1.7253749370574951,
+      "logps/chosen": -389.62030029296875,
+      "logps/rejected": -485.5874938964844,
+      "loss": 0.4941,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.2529852390289307,
+      "rewards/margins": 1.0635504722595215,
+      "rewards/rejected": -2.316535472869873,
       "step": 450
     },
     {
       "epoch": 0.9628466771323915,
-      "grad_norm": 29.402106331456956,
+      "grad_norm": 34.525985232536584,
       "learning_rate": 1.9347820230782295e-09,
-      "logits/chosen": -1.2285839319229126,
-      "logits/rejected": -1.0060263872146606,
-      "logps/chosen": -425.80548095703125,
-      "logps/rejected": -479.5809631347656,
-      "loss": 0.5024,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -1.590386152267456,
-      "rewards/margins": 0.9592040777206421,
-      "rewards/rejected": -2.5495898723602295,
+      "logits/chosen": -1.9128828048706055,
+      "logits/rejected": -1.8068416118621826,
+      "logps/chosen": -388.21502685546875,
+      "logps/rejected": -444.5638732910156,
+      "loss": 0.5012,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.223968744277954,
+      "rewards/margins": 0.9889059066772461,
+      "rewards/rejected": -2.2128746509552,
       "step": 460
     },
     {
       "epoch": 0.9837781266352695,
-      "grad_norm": 27.7268805701989,
+      "grad_norm": 26.516864095070762,
       "learning_rate": 3.2839470889836627e-10,
-      "logits/chosen": -1.2228537797927856,
-      "logits/rejected": -1.0901638269424438,
-      "logps/chosen": -444.19976806640625,
-      "logps/rejected": -506.9326171875,
-      "loss": 0.4792,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -1.5240809917449951,
-      "rewards/margins": 0.9203958511352539,
-      "rewards/rejected": -2.444476842880249,
+      "logits/chosen": -1.9164927005767822,
+      "logits/rejected": -1.8577802181243896,
+      "logps/chosen": -407.23077392578125,
+      "logps/rejected": -462.60791015625,
+      "loss": 0.4888,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.1636872291564941,
+      "rewards/margins": 0.8442845344543457,
+      "rewards/rejected": -2.00797176361084,
       "step": 470
     },
     {
       "epoch": 0.9984301412872841,
-      "eval_logits/chosen": -1.1366699934005737,
-      "eval_logits/rejected": -1.0305088758468628,
-      "eval_logps/chosen": -407.18035888671875,
-      "eval_logps/rejected": -511.866943359375,
-      "eval_loss": 0.4978238046169281,
-      "eval_rewards/accuracies": 0.79296875,
-      "eval_rewards/chosen": -1.5086967945098877,
-      "eval_rewards/margins": 1.0383816957473755,
-      "eval_rewards/rejected": -2.5470783710479736,
-      "eval_runtime": 168.9415,
-      "eval_samples_per_second": 11.838,
-      "eval_steps_per_second": 0.189,
+      "eval_logits/chosen": -1.9287464618682861,
+      "eval_logits/rejected": -1.8791497945785522,
+      "eval_logps/chosen": -368.52239990234375,
+      "eval_logps/rejected": -470.2213439941406,
+      "eval_loss": 0.5001584887504578,
+      "eval_rewards/accuracies": 0.76171875,
+      "eval_rewards/chosen": -1.1341644525527954,
+      "eval_rewards/margins": 1.0070396661758423,
+      "eval_rewards/rejected": -2.1412041187286377,
+      "eval_runtime": 174.7471,
+      "eval_samples_per_second": 11.445,
+      "eval_steps_per_second": 0.183,
       "step": 477
     },
     {
       "epoch": 0.9984301412872841,
       "step": 477,
       "total_flos": 0.0,
-      "train_loss": 0.5411407262774134,
-      "train_runtime": 15368.6971,
-      "train_samples_per_second": 3.978,
+      "train_loss": 0.5381635759861464,
+      "train_runtime": 15576.9752,
+      "train_samples_per_second": 3.925,
       "train_steps_per_second": 0.031
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4383be6f0c55e9b8ad4d8afe8fb1de35167908d57dd30f3e79c3e4dc0bdb5638
+oid sha256:5ef94a6b63a9657cbf82fb3c915c3feb3a234c888149445f48b558c3307342dc
 size 7672