Model save

README.md

@@ -16,15 +16,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model was trained from scratch on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.5398
-- Rewards/chosen: -0.9741
-- Rewards/rejected: -1.7261
-- Rewards/accuracies: 0.7734
-- Rewards/margins: 0.7520
-- Logps/rejected: -444.0901
-- Logps/chosen: -372.8936
-- Logits/rejected: -2.4406
-- Logits/chosen: -2.4512
+- Loss: 0.5332
+- Rewards/chosen: -0.5681
+- Rewards/rejected: -1.3276
+- Rewards/accuracies: 0.7695
+- Rewards/margins: 0.7596
+- Logps/rejected: -420.6561
+- Logps/chosen: -342.5578
+- Logits/rejected: -0.2657
+- Logits/chosen: -0.4442
 
 ## Model description
 
@@ -44,12 +44,12 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 5e-07
-- train_batch_size: 8
+- train_batch_size: 4
 - eval_batch_size: 8
 - seed: 42
 - distributed_type: multi-GPU
 - num_devices: 8
-- gradient_accumulation_steps: 2
+- gradient_accumulation_steps: 4
 - total_train_batch_size: 128
 - total_eval_batch_size: 64
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
@@ -61,15 +61,12 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.6101        | 0.2092 | 100  | 0.6109          | -0.4573        | -0.8177          | 0.7266             | 0.3604          | -353.2573      | -321.2165    | -2.6231         | -2.6442       |
-| 0.5706        | 0.4184 | 200  | 0.5637          | -0.7020        | -1.2969          | 0.75               | 0.5949          | -401.1722      | -345.6816    | -2.5773         | -2.5921       |
-| 0.5242        | 0.6276 | 300  | 0.5492          | -0.8252        | -1.5001          | 0.7617             | 0.6750          | -421.4963      | -357.9999    | -2.4884         | -2.5005       |
-| 0.5352        | 0.8368 | 400  | 0.5398          | -0.9741        | -1.7261          | 0.7734             | 0.7520          | -444.0901      | -372.8936    | -2.4406         | -2.4512       |
+| 0.5164        | 0.9984 | 477  | 0.5332          | -0.5681        | -1.3276          | 0.7695             | 0.7596          | -420.6561      | -342.5578    | -0.2657         | -0.4442       |
 
 
 ### Framework versions
 
 - Transformers 4.45.2
-- Pytorch 2.2.2+rocm5.7
-- Datasets 3.2.0
+- Pytorch 2.5.1+rocm6.2
+- Datasets 3.5.0
 - Tokenizers 0.20.3

all_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 1.0,
+    "epoch": 0.9984301412872841,
     "total_flos": 0.0,
-    "train_loss": 0.5718294187569718,
-    "train_runtime": 20140.7322,
+    "train_loss": 0.5657099517886244,
+    "train_runtime": 14255.2941,
     "train_samples": 61134,
-    "train_samples_per_second": 3.035,
-    "train_steps_per_second": 0.024
+    "train_samples_per_second": 4.289,
+    "train_steps_per_second": 0.033
 }

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "data/zephyr-7b-sft-full/checkpoint-800",
+  "_name_or_path": "data/zephyr-7b-sft-full/checkpoint-600",
   "architectures": [
     "MistralForCausalLM"
   ],

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5ee55e43c7881e8e1bddcde2550a7d899f28d917042ac84d94d1ebf3c4e18a1a
+oid sha256:6cbac3133b4079630e5dda36990174ef15873b65e5dbdbc2a036f7bed44f96f1
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:00c2611f7f7ed5a9f0c31bb8a26c6a972f636fb194a37042cde60fb9c6d5eb1d
+oid sha256:ee3952ec5a836ccba125ccf78b64940c7610b3847cbe0dd2893bbb0a0ff9d946
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:26a75b3eb08cc3cbf8323b260059b40d6b90da9b404aace4c15978a9f3dd08f8
+oid sha256:59847a7c96474474716c6a13874c949426159cf768407a8deb4cbdd7747e95c1
 size 4540516344

train_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 1.0,
+    "epoch": 0.9984301412872841,
     "total_flos": 0.0,
-    "train_loss": 0.5718294187569718,
-    "train_runtime": 20140.7322,
+    "train_loss": 0.5657099517886244,
+    "train_runtime": 14255.2941,
     "train_samples": 61134,
-    "train_samples_per_second": 3.035,
-    "train_steps_per_second": 0.024
+    "train_samples_per_second": 4.289,
+    "train_steps_per_second": 0.033
 }

trainer_state.json

@@ -1,21 +1,21 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.0,
-  "eval_steps": 100,
-  "global_step": 478,
+  "epoch": 0.9984301412872841,
+  "eval_steps": 500,
+  "global_step": 477,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.0020920502092050207,
-      "grad_norm": 8.757903324169646,
+      "epoch": 0.0020931449502878076,
+      "grad_norm": 18.07411787458952,
       "learning_rate": 1.0416666666666666e-08,
-      "logits/chosen": -2.6758785247802734,
-      "logits/rejected": -2.6680283546447754,
-      "logps/chosen": -282.2353820800781,
-      "logps/rejected": -371.613525390625,
+      "logits/chosen": -2.6982662677764893,
+      "logits/rejected": -2.639620304107666,
+      "logps/chosen": -340.48919677734375,
+      "logps/rejected": -383.30560302734375,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -24,789 +24,741 @@
       "step": 1
     },
     {
-      "epoch": 0.02092050209205021,
-      "grad_norm": 8.677585116413498,
+      "epoch": 0.020931449502878074,
+      "grad_norm": 9.9608972008456,
       "learning_rate": 1.0416666666666667e-07,
-      "logits/chosen": -2.5242655277252197,
-      "logits/rejected": -2.5032453536987305,
-      "logps/chosen": -272.8685302734375,
-      "logps/rejected": -259.168701171875,
-      "loss": 0.6931,
-      "rewards/accuracies": 0.4305555522441864,
-      "rewards/chosen": 0.00020810285059269518,
-      "rewards/margins": 0.0003322906559333205,
-      "rewards/rejected": -0.0001241877762367949,
+      "logits/chosen": -2.4643380641937256,
+      "logits/rejected": -2.41880464553833,
+      "logps/chosen": -332.1871032714844,
+      "logps/rejected": -297.4084167480469,
+      "loss": 0.693,
+      "rewards/accuracies": 0.5277777910232544,
+      "rewards/chosen": 0.0005977663677185774,
+      "rewards/margins": 0.0007145923445932567,
+      "rewards/rejected": -0.00011682594777084887,
       "step": 10
     },
     {
-      "epoch": 0.04184100418410042,
-      "grad_norm": 7.752266910239108,
+      "epoch": 0.04186289900575615,
+      "grad_norm": 13.745638539501607,
       "learning_rate": 2.0833333333333333e-07,
-      "logits/chosen": -2.576795816421509,
-      "logits/rejected": -2.5414347648620605,
-      "logps/chosen": -290.27386474609375,
-      "logps/rejected": -302.90447998046875,
-      "loss": 0.6928,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.0002451314067002386,
-      "rewards/margins": 0.0005731090204790235,
-      "rewards/rejected": -0.00032797764288261533,
+      "logits/chosen": -2.563354969024658,
+      "logits/rejected": -2.4802138805389404,
+      "logps/chosen": -330.6456604003906,
+      "logps/rejected": -303.16796875,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.0026146548334509134,
+      "rewards/margins": 0.0005810384755022824,
+      "rewards/rejected": 0.0020336161833256483,
       "step": 20
     },
     {
-      "epoch": 0.06276150627615062,
-      "grad_norm": 7.952201982598315,
+      "epoch": 0.06279434850863422,
+      "grad_norm": 7.832516309339627,
       "learning_rate": 3.1249999999999997e-07,
-      "logits/chosen": -2.5962252616882324,
-      "logits/rejected": -2.5463669300079346,
-      "logps/chosen": -313.28192138671875,
-      "logps/rejected": -273.99127197265625,
-      "loss": 0.6911,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": 0.003021394833922386,
-      "rewards/margins": 0.004253971856087446,
-      "rewards/rejected": -0.0012325774878263474,
+      "logits/chosen": -2.532294511795044,
+      "logits/rejected": -2.4404759407043457,
+      "logps/chosen": -312.492431640625,
+      "logps/rejected": -266.53936767578125,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.01948517933487892,
+      "rewards/margins": 0.006482087075710297,
+      "rewards/rejected": 0.0130030931904912,
       "step": 30
     },
     {
-      "epoch": 0.08368200836820083,
-      "grad_norm": 7.764909269243244,
+      "epoch": 0.0837257980115123,
+      "grad_norm": 7.9616158798093855,
       "learning_rate": 4.1666666666666667e-07,
-      "logits/chosen": -2.525210380554199,
-      "logits/rejected": -2.505263328552246,
-      "logps/chosen": -271.3431396484375,
-      "logps/rejected": -251.4092559814453,
-      "loss": 0.6857,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.010489744134247303,
-      "rewards/margins": 0.019626062363386154,
-      "rewards/rejected": -0.009136319160461426,
+      "logits/chosen": -2.431403636932373,
+      "logits/rejected": -2.379568099975586,
+      "logps/chosen": -308.171630859375,
+      "logps/rejected": -279.072998046875,
+      "loss": 0.6845,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.046426668763160706,
+      "rewards/margins": 0.019384978339076042,
+      "rewards/rejected": 0.027041692286729813,
       "step": 40
     },
     {
-      "epoch": 0.10460251046025104,
-      "grad_norm": 7.84572317927612,
-      "learning_rate": 4.999733114418725e-07,
-      "logits/chosen": -2.463291645050049,
-      "logits/rejected": -2.436424732208252,
-      "logps/chosen": -292.8009948730469,
-      "logps/rejected": -301.97540283203125,
-      "loss": 0.6779,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": 0.006917999591678381,
-      "rewards/margins": 0.03536984324455261,
-      "rewards/rejected": -0.028451839461922646,
+      "epoch": 0.10465724751439037,
+      "grad_norm": 12.138395522818126,
+      "learning_rate": 4.999731868769026e-07,
+      "logits/chosen": -2.467078685760498,
+      "logits/rejected": -2.3770642280578613,
+      "logps/chosen": -292.5061950683594,
+      "logps/rejected": -287.78692626953125,
+      "loss": 0.6768,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.05952076241374016,
+      "rewards/margins": 0.031759217381477356,
+      "rewards/rejected": 0.027761543169617653,
       "step": 50
     },
     {
-      "epoch": 0.12552301255230125,
-      "grad_norm": 8.99044244558934,
-      "learning_rate": 4.990398100856366e-07,
-      "logits/chosen": -2.5300676822662354,
-      "logits/rejected": -2.491748332977295,
-      "logps/chosen": -286.4462585449219,
-      "logps/rejected": -313.32476806640625,
-      "loss": 0.6654,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.012607897631824017,
-      "rewards/margins": 0.05728869512677193,
-      "rewards/rejected": -0.06989657878875732,
+      "epoch": 0.12558869701726844,
+      "grad_norm": 12.71284691980484,
+      "learning_rate": 4.990353313429303e-07,
+      "logits/chosen": -2.5058963298797607,
+      "logits/rejected": -2.4367785453796387,
+      "logps/chosen": -269.9830322265625,
+      "logps/rejected": -261.7590026855469,
+      "loss": 0.6648,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.054543137550354004,
+      "rewards/margins": 0.06885507702827454,
+      "rewards/rejected": -0.01431194506585598,
       "step": 60
     },
     {
-      "epoch": 0.14644351464435146,
-      "grad_norm": 8.75014782833078,
-      "learning_rate": 4.967775735898179e-07,
-      "logits/chosen": -2.460754156112671,
-      "logits/rejected": -2.4715116024017334,
-      "logps/chosen": -280.77801513671875,
-      "logps/rejected": -283.21710205078125,
-      "loss": 0.6445,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -0.0388030931353569,
-      "rewards/margins": 0.12870901823043823,
-      "rewards/rejected": -0.16751211881637573,
+      "epoch": 0.14652014652014653,
+      "grad_norm": 8.977266411290348,
+      "learning_rate": 4.967625656594781e-07,
+      "logits/chosen": -2.4574174880981445,
+      "logits/rejected": -2.4204437732696533,
+      "logps/chosen": -323.3358459472656,
+      "logps/rejected": -313.7773742675781,
+      "loss": 0.6448,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.02133224718272686,
+      "rewards/margins": 0.09954627603292465,
+      "rewards/rejected": -0.12087850272655487,
       "step": 70
     },
     {
-      "epoch": 0.16736401673640167,
-      "grad_norm": 12.288326071921293,
-      "learning_rate": 4.931986719649298e-07,
-      "logits/chosen": -2.5984039306640625,
-      "logits/rejected": -2.564687967300415,
-      "logps/chosen": -358.021484375,
-      "logps/rejected": -307.74676513671875,
-      "loss": 0.6313,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -0.18460232019424438,
-      "rewards/margins": 0.16194236278533936,
-      "rewards/rejected": -0.34654468297958374,
+      "epoch": 0.1674515960230246,
+      "grad_norm": 15.44090832738,
+      "learning_rate": 4.93167072587771e-07,
+      "logits/chosen": -2.5686161518096924,
+      "logits/rejected": -2.459707260131836,
+      "logps/chosen": -367.8087463378906,
+      "logps/rejected": -306.7381286621094,
+      "loss": 0.6262,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.07196881622076035,
+      "rewards/margins": 0.20668184757232666,
+      "rewards/rejected": -0.2786506414413452,
       "step": 80
     },
     {
-      "epoch": 0.18828451882845187,
-      "grad_norm": 11.854500663971278,
-      "learning_rate": 4.883222001996351e-07,
-      "logits/chosen": -2.55795955657959,
-      "logits/rejected": -2.5331249237060547,
-      "logps/chosen": -306.12774658203125,
-      "logps/rejected": -327.48291015625,
-      "loss": 0.6103,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.37835103273391724,
-      "rewards/margins": 0.29932525753974915,
-      "rewards/rejected": -0.6776763200759888,
+      "epoch": 0.18838304552590268,
+      "grad_norm": 17.357813817512778,
+      "learning_rate": 4.882681251368548e-07,
+      "logits/chosen": -2.484760284423828,
+      "logits/rejected": -2.4333367347717285,
+      "logps/chosen": -286.19573974609375,
+      "logps/rejected": -318.17987060546875,
+      "loss": 0.6072,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.13759762048721313,
+      "rewards/margins": 0.2588348686695099,
+      "rewards/rejected": -0.396432489156723,
       "step": 90
     },
     {
-      "epoch": 0.20920502092050208,
-      "grad_norm": 10.506808977287172,
-      "learning_rate": 4.821741763807186e-07,
-      "logits/chosen": -2.574136972427368,
-      "logits/rejected": -2.570249557495117,
-      "logps/chosen": -315.4927673339844,
-      "logps/rejected": -346.36273193359375,
-      "loss": 0.6101,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.45679885149002075,
-      "rewards/margins": 0.35896575450897217,
-      "rewards/rejected": -0.8157645463943481,
-      "step": 100
-    },
-    {
-      "epoch": 0.20920502092050208,
-      "eval_logits/chosen": -2.6441538333892822,
-      "eval_logits/rejected": -2.623080015182495,
-      "eval_logps/chosen": -321.21649169921875,
-      "eval_logps/rejected": -353.25726318359375,
-      "eval_loss": 0.6109189987182617,
-      "eval_rewards/accuracies": 0.7265625,
-      "eval_rewards/chosen": -0.45733410120010376,
-      "eval_rewards/margins": 0.36040300130844116,
-      "eval_rewards/rejected": -0.8177371025085449,
-      "eval_runtime": 205.7366,
-      "eval_samples_per_second": 9.721,
-      "eval_steps_per_second": 0.156,
+      "epoch": 0.20931449502878074,
+      "grad_norm": 11.555230239819865,
+      "learning_rate": 4.820919832540181e-07,
+      "logits/chosen": -2.481173038482666,
+      "logits/rejected": -2.436263084411621,
+      "logps/chosen": -343.57391357421875,
+      "logps/rejected": -353.5834655761719,
+      "loss": 0.6047,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.17556986212730408,
+      "rewards/margins": 0.37368106842041016,
+      "rewards/rejected": -0.5492509603500366,
       "step": 100
     },
     {
-      "epoch": 0.2301255230125523,
-      "grad_norm": 12.851178130092231,
-      "learning_rate": 4.747874028753375e-07,
-      "logits/chosen": -2.6588165760040283,
-      "logits/rejected": -2.626171827316284,
-      "logps/chosen": -365.60186767578125,
-      "logps/rejected": -342.35577392578125,
-      "loss": 0.6111,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.4737192988395691,
-      "rewards/margins": 0.30567842721939087,
-      "rewards/rejected": -0.7793977856636047,
+      "epoch": 0.2302459445316588,
+      "grad_norm": 12.709794889392892,
+      "learning_rate": 4.7467175306295647e-07,
+      "logits/chosen": -2.3503360748291016,
+      "logits/rejected": -2.259538173675537,
+      "logps/chosen": -345.1012268066406,
+      "logps/rejected": -358.5026550292969,
+      "loss": 0.6012,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.3211168348789215,
+      "rewards/margins": 0.30802685022354126,
+      "rewards/rejected": -0.6291436553001404,
       "step": 110
     },
     {
-      "epoch": 0.2510460251046025,
-      "grad_norm": 11.812990475216695,
-      "learning_rate": 4.662012913161997e-07,
-      "logits/chosen": -2.5753915309906006,
-      "logits/rejected": -2.5762243270874023,
-      "logps/chosen": -340.4606018066406,
-      "logps/rejected": -355.8326110839844,
-      "loss": 0.584,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.48977130651474,
-      "rewards/margins": 0.3774973154067993,
-      "rewards/rejected": -0.8672686815261841,
+      "epoch": 0.25117739403453687,
+      "grad_norm": 16.24550776128439,
+      "learning_rate": 4.6604720940421207e-07,
+      "logits/chosen": -1.8585760593414307,
+      "logits/rejected": -1.812425971031189,
+      "logps/chosen": -319.6259765625,
+      "logps/rejected": -346.91424560546875,
+      "loss": 0.5874,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.20474886894226074,
+      "rewards/margins": 0.4103702902793884,
+      "rewards/rejected": -0.6151191592216492,
       "step": 120
     },
     {
-      "epoch": 0.2719665271966527,
-      "grad_norm": 13.157592262251784,
-      "learning_rate": 4.5646165232345103e-07,
-      "logits/chosen": -2.6023926734924316,
-      "logits/rejected": -2.606266498565674,
-      "logps/chosen": -349.37554931640625,
-      "logps/rejected": -374.80670166015625,
-      "loss": 0.5759,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.5950827598571777,
-      "rewards/margins": 0.3772185444831848,
-      "rewards/rejected": -0.9723013639450073,
+      "epoch": 0.272108843537415,
+      "grad_norm": 18.658464566773528,
+      "learning_rate": 4.5626458262912735e-07,
+      "logits/chosen": -1.4963523149490356,
+      "logits/rejected": -1.3689486980438232,
+      "logps/chosen": -381.5474548339844,
+      "logps/rejected": -392.7295837402344,
+      "loss": 0.5767,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.6478039622306824,
+      "rewards/margins": 0.3431198000907898,
+      "rewards/rejected": -0.9909237623214722,
       "step": 130
     },
     {
-      "epoch": 0.2928870292887029,
-      "grad_norm": 15.344200538370714,
-      "learning_rate": 4.456204510851956e-07,
-      "logits/chosen": -2.612006664276123,
-      "logits/rejected": -2.604703426361084,
-      "logps/chosen": -396.28912353515625,
-      "logps/rejected": -423.3656311035156,
-      "loss": 0.5824,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.6383721232414246,
-      "rewards/margins": 0.458970308303833,
-      "rewards/rejected": -1.0973424911499023,
+      "epoch": 0.29304029304029305,
+      "grad_norm": 14.61768252656386,
+      "learning_rate": 4.453763107901675e-07,
+      "logits/chosen": -1.2942148447036743,
+      "logits/rejected": -1.0670334100723267,
+      "logps/chosen": -374.21197509765625,
+      "logps/rejected": -370.75750732421875,
+      "loss": 0.5762,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.3039678633213043,
+      "rewards/margins": 0.4882018566131592,
+      "rewards/rejected": -0.7921696901321411,
       "step": 140
     },
     {
-      "epoch": 0.3138075313807531,
-      "grad_norm": 19.974569180634063,
-      "learning_rate": 4.337355301007335e-07,
-      "logits/chosen": -2.584885597229004,
-      "logits/rejected": -2.556546688079834,
-      "logps/chosen": -364.377685546875,
-      "logps/rejected": -387.48431396484375,
-      "loss": 0.5806,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.6935727596282959,
-      "rewards/margins": 0.3373980224132538,
-      "rewards/rejected": -1.030970811843872,
+      "epoch": 0.3139717425431711,
+      "grad_norm": 22.141164123200294,
+      "learning_rate": 4.3344075855595097e-07,
+      "logits/chosen": -1.035481572151184,
+      "logits/rejected": -0.8223312497138977,
+      "logps/chosen": -351.9337463378906,
+      "logps/rejected": -350.9710693359375,
+      "loss": 0.5711,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.4184446334838867,
+      "rewards/margins": 0.40214234590530396,
+      "rewards/rejected": -0.8205869793891907,
       "step": 150
     },
     {
-      "epoch": 0.33472803347280333,
-      "grad_norm": 15.705241265957017,
-      "learning_rate": 4.2087030056579986e-07,
-      "logits/chosen": -2.6063337326049805,
-      "logits/rejected": -2.574131488800049,
-      "logps/chosen": -342.39996337890625,
-      "logps/rejected": -362.5370788574219,
-      "loss": 0.5795,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.5668545961380005,
-      "rewards/margins": 0.49046269059181213,
-      "rewards/rejected": -1.0573171377182007,
+      "epoch": 0.3349031920460492,
+      "grad_norm": 16.775320749910513,
+      "learning_rate": 4.2052190435769554e-07,
+      "logits/chosen": -0.8283090591430664,
+      "logits/rejected": -0.4383368492126465,
+      "logps/chosen": -351.2560119628906,
+      "logps/rejected": -372.83636474609375,
+      "loss": 0.5738,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.5103927850723267,
+      "rewards/margins": 0.5579272508621216,
+      "rewards/rejected": -1.0683200359344482,
       "step": 160
     },
     {
-      "epoch": 0.35564853556485354,
-      "grad_norm": 21.508437561410116,
-      "learning_rate": 4.070934040463998e-07,
-      "logits/chosen": -2.541980504989624,
-      "logits/rejected": -2.517639398574829,
-      "logps/chosen": -324.37298583984375,
-      "logps/rejected": -338.1640625,
-      "loss": 0.5723,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.6579991579055786,
-      "rewards/margins": 0.33937641978263855,
-      "rewards/rejected": -0.9973756074905396,
+      "epoch": 0.35583464154892724,
+      "grad_norm": 18.321929450201765,
+      "learning_rate": 4.0668899744407567e-07,
+      "logits/chosen": -0.2253287136554718,
+      "logits/rejected": 0.010831773281097412,
+      "logps/chosen": -337.71563720703125,
+      "logps/rejected": -353.23150634765625,
+      "loss": 0.5681,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.5377098917961121,
+      "rewards/margins": 0.43757739663124084,
+      "rewards/rejected": -0.9752873182296753,
       "step": 170
     },
     {
-      "epoch": 0.37656903765690375,
-      "grad_norm": 18.10631100120152,
-      "learning_rate": 3.9247834624635404e-07,
-      "logits/chosen": -2.5356695652008057,
-      "logits/rejected": -2.5368237495422363,
-      "logps/chosen": -343.92864990234375,
-      "logps/rejected": -355.13916015625,
-      "loss": 0.5484,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.8035238981246948,
-      "rewards/margins": 0.4253202974796295,
-      "rewards/rejected": -1.2288442850112915,
+      "epoch": 0.37676609105180536,
+      "grad_norm": 17.583054376284373,
+      "learning_rate": 3.920161866827889e-07,
+      "logits/chosen": -0.036564283072948456,
+      "logits/rejected": 0.21061678230762482,
+      "logps/chosen": -348.42608642578125,
+      "logps/rejected": -363.5872802734375,
+      "loss": 0.5376,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.5707951784133911,
+      "rewards/margins": 0.45119351148605347,
+      "rewards/rejected": -1.0219886302947998,
       "step": 180
     },
     {
-      "epoch": 0.39748953974895396,
-      "grad_norm": 17.973360155017694,
-      "learning_rate": 3.7710310482256523e-07,
-      "logits/chosen": -2.5625059604644775,
-      "logits/rejected": -2.5411581993103027,
-      "logps/chosen": -351.89971923828125,
-      "logps/rejected": -384.46502685546875,
-      "loss": 0.5665,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -0.6912161111831665,
-      "rewards/margins": 0.4274213910102844,
-      "rewards/rejected": -1.1186374425888062,
+      "epoch": 0.3976975405546834,
+      "grad_norm": 23.074661957066084,
+      "learning_rate": 3.765821230985757e-07,
+      "logits/chosen": 0.14501239359378815,
+      "logits/rejected": 0.1892128884792328,
+      "logps/chosen": -327.35870361328125,
+      "logps/rejected": -371.1208801269531,
+      "loss": 0.5554,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.48090630769729614,
+      "rewards/margins": 0.45326748490333557,
+      "rewards/rejected": -0.9341737627983093,
       "step": 190
     },
     {
-      "epoch": 0.41841004184100417,
-      "grad_norm": 105.1959917368864,
-      "learning_rate": 3.610497133404795e-07,
-      "logits/chosen": -2.517104148864746,
-      "logits/rejected": -2.532656669616699,
-      "logps/chosen": -351.03179931640625,
-      "logps/rejected": -398.70452880859375,
-      "loss": 0.5706,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.8004492521286011,
-      "rewards/margins": 0.5517168641090393,
-      "rewards/rejected": -1.3521660566329956,
-      "step": 200
-    },
-    {
-      "epoch": 0.41841004184100417,
-      "eval_logits/chosen": -2.5920896530151367,
-      "eval_logits/rejected": -2.5773017406463623,
-      "eval_logps/chosen": -345.681640625,
-      "eval_logps/rejected": -401.1722412109375,
-      "eval_loss": 0.5636610388755798,
-      "eval_rewards/accuracies": 0.75,
-      "eval_rewards/chosen": -0.7019854187965393,
-      "eval_rewards/margins": 0.5949010252952576,
-      "eval_rewards/rejected": -1.2968864440917969,
-      "eval_runtime": 208.2203,
-      "eval_samples_per_second": 9.605,
-      "eval_steps_per_second": 0.154,
+      "epoch": 0.4186289900575615,
+      "grad_norm": 22.069261987026525,
+      "learning_rate": 3.604695382782159e-07,
+      "logits/chosen": 0.5265301465988159,
+      "logits/rejected": 0.6243599653244019,
+      "logps/chosen": -359.69854736328125,
+      "logps/rejected": -418.3558044433594,
+      "loss": 0.5655,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.7841523885726929,
+      "rewards/margins": 0.46337419748306274,
+      "rewards/rejected": -1.2475265264511108,
       "step": 200
     },
     {
-      "epoch": 0.4393305439330544,
-      "grad_norm": 17.52002295184988,
-      "learning_rate": 3.4440382358952115e-07,
-      "logits/chosen": -2.509565830230713,
-      "logits/rejected": -2.4933359622955322,
-      "logps/chosen": -379.9023132324219,
-      "logps/rejected": -380.1143798828125,
-      "loss": 0.5726,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.7864257097244263,
-      "rewards/margins": 0.43172281980514526,
-      "rewards/rejected": -1.2181484699249268,
+      "epoch": 0.43956043956043955,
+      "grad_norm": 17.42416201312016,
+      "learning_rate": 3.4376480090239047e-07,
+      "logits/chosen": -0.36963820457458496,
+      "logits/rejected": 0.10797449201345444,
+      "logps/chosen": -408.05328369140625,
+      "logps/rejected": -385.1399841308594,
+      "loss": 0.573,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.7120479345321655,
+      "rewards/margins": 0.45911431312561035,
+      "rewards/rejected": -1.1711622476577759,
       "step": 210
     },
     {
-      "epoch": 0.4602510460251046,
-      "grad_norm": 20.831804487729595,
-      "learning_rate": 3.272542485937368e-07,
-      "logits/chosen": -2.4844298362731934,
-      "logits/rejected": -2.4896388053894043,
-      "logps/chosen": -357.0660705566406,
-      "logps/rejected": -380.40411376953125,
-      "loss": 0.5624,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.8168194890022278,
-      "rewards/margins": 0.533184289932251,
-      "rewards/rejected": -1.3500036001205444,
+      "epoch": 0.4604918890633176,
+      "grad_norm": 21.650857513168543,
+      "learning_rate": 3.265574537815398e-07,
+      "logits/chosen": -1.1279886960983276,
+      "logits/rejected": -1.0101404190063477,
+      "logps/chosen": -330.86920166015625,
+      "logps/rejected": -375.27581787109375,
+      "loss": 0.5521,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.5428093075752258,
+      "rewards/margins": 0.5479553937911987,
+      "rewards/rejected": -1.0907647609710693,
       "step": 220
     },
     {
-      "epoch": 0.4811715481171548,
-      "grad_norm": 20.183953311249173,
-      "learning_rate": 3.096924887558854e-07,
-      "logits/chosen": -2.5007448196411133,
-      "logits/rejected": -2.4937093257904053,
-      "logps/chosen": -352.20819091796875,
-      "logps/rejected": -411.02618408203125,
-      "loss": 0.5632,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.951431155204773,
-      "rewards/margins": 0.600155234336853,
-      "rewards/rejected": -1.5515865087509155,
+      "epoch": 0.48142333856619574,
+      "grad_norm": 21.29968622459755,
+      "learning_rate": 3.0893973387735683e-07,
+      "logits/chosen": -1.4807205200195312,
+      "logits/rejected": -1.2653987407684326,
+      "logps/chosen": -349.1932067871094,
+      "logps/rejected": -402.3145751953125,
+      "loss": 0.5591,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.6056884527206421,
+      "rewards/margins": 0.7848646640777588,
+      "rewards/rejected": -1.3905531167984009,
       "step": 230
     },
     {
-      "epoch": 0.502092050209205,
-      "grad_norm": 20.513014768183723,
-      "learning_rate": 2.9181224366319943e-07,
-      "logits/chosen": -2.5379579067230225,
-      "logits/rejected": -2.5073440074920654,
-      "logps/chosen": -360.53839111328125,
-      "logps/rejected": -397.48583984375,
-      "loss": 0.5317,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.8155630826950073,
-      "rewards/margins": 0.6164010763168335,
-      "rewards/rejected": -1.4319641590118408,
+      "epoch": 0.5023547880690737,
+      "grad_norm": 21.54037562544609,
+      "learning_rate": 2.910060778827554e-07,
+      "logits/chosen": -1.4609298706054688,
+      "logits/rejected": -1.2633426189422607,
+      "logps/chosen": -361.18109130859375,
+      "logps/rejected": -385.7818298339844,
+      "loss": 0.5293,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.4389161169528961,
+      "rewards/margins": 0.5328649878501892,
+      "rewards/rejected": -0.9717811346054077,
       "step": 240
     },
     {
-      "epoch": 0.5230125523012552,
-      "grad_norm": 21.446368846425955,
-      "learning_rate": 2.7370891215954565e-07,
-      "logits/chosen": -2.4896950721740723,
-      "logits/rejected": -2.465409755706787,
-      "logps/chosen": -398.04925537109375,
-      "logps/rejected": -420.305908203125,
-      "loss": 0.5466,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.8571982383728027,
-      "rewards/margins": 0.6700164079666138,
-      "rewards/rejected": -1.527214765548706,
+      "epoch": 0.5232862375719518,
+      "grad_norm": 19.10437589239632,
+      "learning_rate": 2.7285261601056697e-07,
+      "logits/chosen": -1.1038726568222046,
+      "logits/rejected": -0.7872940897941589,
+      "logps/chosen": -367.0468444824219,
+      "logps/rejected": -402.64837646484375,
+      "loss": 0.5392,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -0.618547260761261,
+      "rewards/margins": 0.7416694760322571,
+      "rewards/rejected": -1.360216736793518,
       "step": 250
     },
     {
-      "epoch": 0.5439330543933054,
-      "grad_norm": 24.82188036781192,
-      "learning_rate": 2.55479083351317e-07,
-      "logits/chosen": -2.525885581970215,
-      "logits/rejected": -2.531085252761841,
-      "logps/chosen": -387.8109436035156,
-      "logps/rejected": -408.15093994140625,
-      "loss": 0.5443,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.8237077593803406,
-      "rewards/margins": 0.6463375091552734,
-      "rewards/rejected": -1.4700452089309692,
+      "epoch": 0.54421768707483,
+      "grad_norm": 27.812573286232187,
+      "learning_rate": 2.5457665670441937e-07,
+      "logits/chosen": -0.946252703666687,
+      "logits/rejected": -0.8400171399116516,
+      "logps/chosen": -360.14691162109375,
+      "logps/rejected": -391.15576171875,
+      "loss": 0.5386,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.7270759344100952,
+      "rewards/margins": 0.5733479261398315,
+      "rewards/rejected": -1.3004238605499268,
       "step": 260
     },
     {
-      "epoch": 0.5648535564853556,
-      "grad_norm": 19.977217937312748,
-      "learning_rate": 2.3722002126275822e-07,
-      "logits/chosen": -2.5017404556274414,
-      "logits/rejected": -2.494499921798706,
-      "logps/chosen": -376.91650390625,
-      "logps/rejected": -404.3011169433594,
-      "loss": 0.5446,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.8682349920272827,
-      "rewards/margins": 0.5327897667884827,
-      "rewards/rejected": -1.4010246992111206,
+      "epoch": 0.565149136577708,
+      "grad_norm": 19.996060331745692,
+      "learning_rate": 2.3627616503391812e-07,
+      "logits/chosen": -1.0181360244750977,
+      "logits/rejected": -0.8419178128242493,
+      "logps/chosen": -381.3074645996094,
+      "logps/rejected": -421.84503173828125,
+      "loss": 0.5439,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.6155784726142883,
+      "rewards/margins": 0.7110739946365356,
+      "rewards/rejected": -1.3266522884368896,
       "step": 270
     },
     {
-      "epoch": 0.5857740585774058,
-      "grad_norm": 23.01702137853262,
-      "learning_rate": 2.19029145890313e-07,
-      "logits/chosen": -2.467871904373169,
-      "logits/rejected": -2.4517643451690674,
-      "logps/chosen": -365.51580810546875,
-      "logps/rejected": -404.5898742675781,
-      "loss": 0.5556,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.8867861032485962,
-      "rewards/margins": 0.6364314556121826,
-      "rewards/rejected": -1.5232175588607788,
+      "epoch": 0.5860805860805861,
+      "grad_norm": 20.456176579909233,
+      "learning_rate": 2.1804923757009882e-07,
+      "logits/chosen": -0.8568949699401855,
+      "logits/rejected": -0.6260634660720825,
+      "logps/chosen": -355.68487548828125,
+      "logps/rejected": -365.54150390625,
+      "loss": 0.5521,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.5479617118835449,
+      "rewards/margins": 0.554092288017273,
+      "rewards/rejected": -1.1020539999008179,
       "step": 280
     },
     {
-      "epoch": 0.606694560669456,
-      "grad_norm": 22.089064125798412,
-      "learning_rate": 2.0100351342479216e-07,
-      "logits/chosen": -2.4908652305603027,
-      "logits/rejected": -2.478694438934326,
-      "logps/chosen": -344.6084289550781,
-      "logps/rejected": -386.208740234375,
-      "loss": 0.5504,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.8696595430374146,
-      "rewards/margins": 0.531941294670105,
-      "rewards/rejected": -1.40160071849823,
+      "epoch": 0.6070120355834642,
+      "grad_norm": 21.855087428817388,
+      "learning_rate": 1.9999357655598891e-07,
+      "logits/chosen": -0.9505099058151245,
+      "logits/rejected": -0.7808119654655457,
+      "logps/chosen": -334.37713623046875,
+      "logps/rejected": -388.1415100097656,
+      "loss": 0.5459,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.6734660863876343,
+      "rewards/margins": 0.5256973505020142,
+      "rewards/rejected": -1.1991634368896484,
       "step": 290
     },
     {
-      "epoch": 0.6276150627615062,
-      "grad_norm": 26.103550254512264,
-      "learning_rate": 1.8323929841460178e-07,
-      "logits/chosen": -2.4707038402557373,
-      "logits/rejected": -2.4599740505218506,
-      "logps/chosen": -401.29876708984375,
-      "logps/rejected": -412.0240173339844,
-      "loss": 0.5242,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.9054401516914368,
-      "rewards/margins": 0.5415455102920532,
-      "rewards/rejected": -1.4469856023788452,
-      "step": 300
-    },
-    {
-      "epoch": 0.6276150627615062,
-      "eval_logits/chosen": -2.500545024871826,
-      "eval_logits/rejected": -2.48840594291687,
-      "eval_logps/chosen": -357.9999084472656,
-      "eval_logps/rejected": -421.496337890625,
-      "eval_loss": 0.5492461919784546,
-      "eval_rewards/accuracies": 0.76171875,
-      "eval_rewards/chosen": -0.8251683712005615,
-      "eval_rewards/margins": 0.674959123134613,
-      "eval_rewards/rejected": -1.5001275539398193,
-      "eval_runtime": 207.1784,
-      "eval_samples_per_second": 9.654,
-      "eval_steps_per_second": 0.154,
+      "epoch": 0.6279434850863422,
+      "grad_norm": 20.553743451672336,
+      "learning_rate": 1.8220596619089573e-07,
+      "logits/chosen": -0.9063513875007629,
+      "logits/rejected": -0.6456217765808105,
+      "logps/chosen": -422.3011169433594,
+      "logps/rejected": -447.60308837890625,
+      "loss": 0.5201,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.6219363212585449,
+      "rewards/margins": 0.6523629426956177,
+      "rewards/rejected": -1.2742993831634521,
       "step": 300
     },
     {
-      "epoch": 0.6485355648535565,
-      "grad_norm": 23.74321791985463,
-      "learning_rate": 1.6583128063291573e-07,
-      "logits/chosen": -2.395718812942505,
-      "logits/rejected": -2.419067859649658,
-      "logps/chosen": -399.44390869140625,
-      "logps/rejected": -425.00250244140625,
-      "loss": 0.5312,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.9401062726974487,
-      "rewards/margins": 0.6394140720367432,
-      "rewards/rejected": -1.5795202255249023,
+      "epoch": 0.6488749345892203,
+      "grad_norm": 19.16702342627125,
+      "learning_rate": 1.647817538357072e-07,
+      "logits/chosen": -0.8550491333007812,
+      "logits/rejected": -0.6107693910598755,
+      "logps/chosen": -392.9826354980469,
+      "logps/rejected": -404.90277099609375,
+      "loss": 0.5161,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.6534227132797241,
+      "rewards/margins": 0.7227972149848938,
+      "rewards/rejected": -1.3762199878692627,
       "step": 310
     },
     {
-      "epoch": 0.6694560669456067,
-      "grad_norm": 23.984504028653255,
-      "learning_rate": 1.488723393865766e-07,
-      "logits/chosen": -2.4724831581115723,
-      "logits/rejected": -2.469027280807495,
-      "logps/chosen": -396.5765380859375,
-      "logps/rejected": -407.79754638671875,
-      "loss": 0.5245,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.9122316241264343,
-      "rewards/margins": 0.6226523518562317,
-      "rewards/rejected": -1.534883975982666,
+      "epoch": 0.6698063840920984,
+      "grad_norm": 20.684056407375124,
+      "learning_rate": 1.478143389201113e-07,
+      "logits/chosen": -0.7066868543624878,
+      "logits/rejected": -0.4419567584991455,
+      "logps/chosen": -337.99554443359375,
+      "logps/rejected": -378.85528564453125,
+      "loss": 0.5178,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -0.5666108131408691,
+      "rewards/margins": 0.76667320728302,
+      "rewards/rejected": -1.3332840204238892,
       "step": 320
     },
     {
-      "epoch": 0.6903765690376569,
-      "grad_norm": 26.046974609360653,
-      "learning_rate": 1.3245295796480788e-07,
-      "logits/chosen": -2.4537134170532227,
-      "logits/rejected": -2.4357807636260986,
-      "logps/chosen": -374.4744873046875,
-      "logps/rejected": -418.5931091308594,
-      "loss": 0.5417,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.9575269818305969,
-      "rewards/margins": 0.518912672996521,
-      "rewards/rejected": -1.4764397144317627,
+      "epoch": 0.6907378335949764,
+      "grad_norm": 23.16918692489251,
+      "learning_rate": 1.3139467229135998e-07,
+      "logits/chosen": -0.9136036038398743,
+      "logits/rejected": -0.8735296130180359,
+      "logps/chosen": -343.11474609375,
+      "logps/rejected": -400.88055419921875,
+      "loss": 0.5291,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.5132071375846863,
+      "rewards/margins": 0.6378787755966187,
+      "rewards/rejected": -1.1510859727859497,
       "step": 330
     },
     {
-      "epoch": 0.7112970711297071,
-      "grad_norm": 32.35920972685731,
-      "learning_rate": 1.1666074087171627e-07,
-      "logits/chosen": -2.476076126098633,
-      "logits/rejected": -2.443582057952881,
-      "logps/chosen": -388.89385986328125,
-      "logps/rejected": -458.06292724609375,
-      "loss": 0.5414,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.874224841594696,
-      "rewards/margins": 0.792776346206665,
-      "rewards/rejected": -1.6670011281967163,
+      "epoch": 0.7116692830978545,
+      "grad_norm": 25.09011052065921,
+      "learning_rate": 1.1561076868822755e-07,
+      "logits/chosen": -0.8220704197883606,
+      "logits/rejected": -0.6656316518783569,
+      "logps/chosen": -376.2585144042969,
+      "logps/rejected": -415.30108642578125,
+      "loss": 0.5298,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.5471745133399963,
+      "rewards/margins": 0.6663635969161987,
+      "rewards/rejected": -1.2135381698608398,
       "step": 340
     },
     {
-      "epoch": 0.7322175732217573,
-      "grad_norm": 21.673059890319784,
-      "learning_rate": 1.0157994641835734e-07,
-      "logits/chosen": -2.4362919330596924,
-      "logits/rejected": -2.422833204269409,
-      "logps/chosen": -361.5782165527344,
-      "logps/rejected": -406.13323974609375,
-      "loss": 0.5143,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.9156524538993835,
-      "rewards/margins": 0.6201896667480469,
-      "rewards/rejected": -1.5358420610427856,
+      "epoch": 0.7326007326007326,
+      "grad_norm": 23.913230474173087,
+      "learning_rate": 1.0054723495346482e-07,
+      "logits/chosen": -0.8336887359619141,
+      "logits/rejected": -0.5391756892204285,
+      "logps/chosen": -333.2408752441406,
+      "logps/rejected": -374.46160888671875,
+      "loss": 0.5046,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.46794453263282776,
+      "rewards/margins": 0.6849315762519836,
+      "rewards/rejected": -1.1528760194778442,
       "step": 350
     },
     {
-      "epoch": 0.7531380753138075,
-      "grad_norm": 21.006664770026205,
-      "learning_rate": 8.729103716819111e-08,
-      "logits/chosen": -2.482931137084961,
-      "logits/rejected": -2.43414306640625,
-      "logps/chosen": -412.705322265625,
-      "logps/rejected": -427.495849609375,
-      "loss": 0.5556,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.9682540893554688,
-      "rewards/margins": 0.6217423677444458,
-      "rewards/rejected": -1.589996337890625,
+      "epoch": 0.7535321821036107,
+      "grad_norm": 18.671801968846303,
+      "learning_rate": 8.628481651367875e-08,
+      "logits/chosen": -0.6670598983764648,
+      "logits/rejected": -0.3578875660896301,
+      "logps/chosen": -389.22222900390625,
+      "logps/rejected": -422.08209228515625,
+      "loss": 0.5407,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.5574816465377808,
+      "rewards/margins": 0.730495035648346,
+      "rewards/rejected": -1.287976861000061,
       "step": 360
     },
     {
-      "epoch": 0.7740585774058577,
-      "grad_norm": 21.069321884286477,
-      "learning_rate": 7.387025063449081e-08,
-      "logits/chosen": -2.4411327838897705,
-      "logits/rejected": -2.4287843704223633,
-      "logps/chosen": -385.64788818359375,
-      "logps/rejected": -402.4495544433594,
-      "loss": 0.5373,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -1.0147379636764526,
-      "rewards/margins": 0.5490932464599609,
-      "rewards/rejected": -1.563831090927124,
+      "epoch": 0.7744636316064888,
+      "grad_norm": 34.13531738812179,
+      "learning_rate": 7.289996455765748e-08,
+      "logits/chosen": -0.6234959363937378,
+      "logits/rejected": -0.37343746423721313,
+      "logps/chosen": -344.8446960449219,
+      "logps/rejected": -383.56939697265625,
+      "loss": 0.5266,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.540058434009552,
+      "rewards/margins": 0.757440984249115,
+      "rewards/rejected": -1.297499418258667,
       "step": 370
     },
     {
-      "epoch": 0.7949790794979079,
-      "grad_norm": 21.84294519368066,
-      "learning_rate": 6.138919252022435e-08,
-      "logits/chosen": -2.3698017597198486,
-      "logits/rejected": -2.3893239498138428,
-      "logps/chosen": -356.11199951171875,
-      "logps/rejected": -441.1004333496094,
-      "loss": 0.5364,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -1.0883876085281372,
-      "rewards/margins": 0.6846298575401306,
-      "rewards/rejected": -1.7730175256729126,
+      "epoch": 0.7953950811093669,
+      "grad_norm": 21.95611942170756,
+      "learning_rate": 6.046442623320145e-08,
+      "logits/chosen": -0.3392348885536194,
+      "logits/rejected": -0.2354392111301422,
+      "logps/chosen": -356.6414794921875,
+      "logps/rejected": -457.1575622558594,
+      "loss": 0.5252,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.7903109788894653,
+      "rewards/margins": 0.8445862531661987,
+      "rewards/rejected": -1.634897232055664,
       "step": 380
     },
     {
-      "epoch": 0.8158995815899581,
-      "grad_norm": 22.21133169200077,
-      "learning_rate": 4.991445467064689e-08,
-      "logits/chosen": -2.417219638824463,
-      "logits/rejected": -2.4007370471954346,
-      "logps/chosen": -413.8736877441406,
-      "logps/rejected": -462.4320373535156,
-      "loss": 0.521,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.9978424906730652,
-      "rewards/margins": 0.6709398031234741,
-      "rewards/rejected": -1.6687822341918945,
+      "epoch": 0.8163265306122449,
+      "grad_norm": 25.808898227359457,
+      "learning_rate": 4.904486005914027e-08,
+      "logits/chosen": -0.7156884074211121,
+      "logits/rejected": -0.4770375192165375,
+      "logps/chosen": -443.89642333984375,
+      "logps/rejected": -474.5353088378906,
+      "loss": 0.5136,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.6931928396224976,
+      "rewards/margins": 0.6563575863838196,
+      "rewards/rejected": -1.349550485610962,
       "step": 390
     },
     {
-      "epoch": 0.8368200836820083,
-      "grad_norm": 28.00908990981185,
-      "learning_rate": 3.9507259776993954e-08,
-      "logits/chosen": -2.4108476638793945,
-      "logits/rejected": -2.3888325691223145,
-      "logps/chosen": -392.48345947265625,
-      "logps/rejected": -451.7422790527344,
-      "loss": 0.5352,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.0856629610061646,
-      "rewards/margins": 0.6859577894210815,
-      "rewards/rejected": -1.771620750427246,
-      "step": 400
-    },
-    {
-      "epoch": 0.8368200836820083,
-      "eval_logits/chosen": -2.451232671737671,
-      "eval_logits/rejected": -2.440563678741455,
-      "eval_logps/chosen": -372.89361572265625,
-      "eval_logps/rejected": -444.0900573730469,
-      "eval_loss": 0.5397807955741882,
-      "eval_rewards/accuracies": 0.7734375,
-      "eval_rewards/chosen": -0.9741055369377136,
-      "eval_rewards/margins": 0.7519596219062805,
-      "eval_rewards/rejected": -1.7260651588439941,
-      "eval_runtime": 210.6901,
-      "eval_samples_per_second": 9.493,
-      "eval_steps_per_second": 0.152,
+      "epoch": 0.837257980115123,
+      "grad_norm": 20.792087565548005,
+      "learning_rate": 3.8702478614051345e-08,
+      "logits/chosen": -0.577874481678009,
+      "logits/rejected": -0.3143076002597809,
+      "logps/chosen": -343.8722839355469,
+      "logps/rejected": -391.06610107421875,
+      "loss": 0.528,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.6408818960189819,
+      "rewards/margins": 0.6500493884086609,
+      "rewards/rejected": -1.290931224822998,
       "step": 400
     },
     {
-      "epoch": 0.8577405857740585,
-      "grad_norm": 27.01620619392976,
-      "learning_rate": 3.022313472693447e-08,
-      "logits/chosen": -2.446744441986084,
-      "logits/rejected": -2.4310455322265625,
-      "logps/chosen": -410.442626953125,
-      "logps/rejected": -448.6288146972656,
-      "loss": 0.5429,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -1.0149585008621216,
-      "rewards/margins": 0.7070528864860535,
-      "rewards/rejected": -1.7220113277435303,
+      "epoch": 0.858189429618001,
+      "grad_norm": 24.33600208096771,
+      "learning_rate": 2.9492720416985e-08,
+      "logits/chosen": -0.859704852104187,
+      "logits/rejected": -0.572528600692749,
+      "logps/chosen": -388.897705078125,
+      "logps/rejected": -415.8624572753906,
+      "loss": 0.5399,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.6071329712867737,
+      "rewards/margins": 0.7060071229934692,
+      "rewards/rejected": -1.3131401538848877,
       "step": 410
     },
     {
-      "epoch": 0.8786610878661087,
-      "grad_norm": 27.366539486392494,
-      "learning_rate": 2.2111614344599684e-08,
-      "logits/chosen": -2.396483898162842,
-      "logits/rejected": -2.4031028747558594,
-      "logps/chosen": -412.63983154296875,
-      "logps/rejected": -452.2643127441406,
-      "loss": 0.5244,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -1.0224273204803467,
-      "rewards/margins": 0.6844695806503296,
-      "rewards/rejected": -1.7068971395492554,
+      "epoch": 0.8791208791208791,
+      "grad_norm": 24.28511601342855,
+      "learning_rate": 2.1464952759020856e-08,
+      "logits/chosen": -0.64429771900177,
+      "logits/rejected": -0.47082147002220154,
+      "logps/chosen": -341.98504638671875,
+      "logps/rejected": -408.1618957519531,
+      "loss": 0.5184,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.6186074614524841,
+      "rewards/margins": 0.6798468828201294,
+      "rewards/rejected": -1.2984542846679688,
       "step": 420
     },
     {
-      "epoch": 0.899581589958159,
-      "grad_norm": 24.875911517489907,
-      "learning_rate": 1.521597710086439e-08,
-      "logits/chosen": -2.3377132415771484,
-      "logits/rejected": -2.3098206520080566,
-      "logps/chosen": -395.1689453125,
-      "logps/rejected": -441.73199462890625,
-      "loss": 0.5186,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.0618338584899902,
-      "rewards/margins": 0.7006598711013794,
-      "rewards/rejected": -1.762493371963501,
+      "epoch": 0.9000523286237572,
+      "grad_norm": 24.32668129564916,
+      "learning_rate": 1.4662207078575684e-08,
+      "logits/chosen": -0.627988338470459,
+      "logits/rejected": -0.37057411670684814,
+      "logps/chosen": -384.9641418457031,
+      "logps/rejected": -431.1815490722656,
+      "loss": 0.5035,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.5622997283935547,
+      "rewards/margins": 0.669585108757019,
+      "rewards/rejected": -1.2318848371505737,
       "step": 430
     },
     {
-      "epoch": 0.9205020920502092,
-      "grad_norm": 21.81838371943632,
-      "learning_rate": 9.57301420397924e-09,
-      "logits/chosen": -2.4200377464294434,
-      "logits/rejected": -2.4004955291748047,
-      "logps/chosen": -392.4845275878906,
-      "logps/rejected": -444.28021240234375,
-      "loss": 0.5264,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.9886407852172852,
-      "rewards/margins": 0.6592715382575989,
-      "rewards/rejected": -1.6479122638702393,
+      "epoch": 0.9209837781266352,
+      "grad_norm": 20.42397496588787,
+      "learning_rate": 9.12094829893642e-09,
+      "logits/chosen": -0.8050533533096313,
+      "logits/rejected": -0.5536060929298401,
+      "logps/chosen": -338.9949035644531,
+      "logps/rejected": -367.10296630859375,
+      "loss": 0.5177,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.5558196306228638,
+      "rewards/margins": 0.6245731711387634,
+      "rewards/rejected": -1.1803929805755615,
       "step": 440
     },
     {
-      "epoch": 0.9414225941422594,
-      "grad_norm": 24.89120124116998,
-      "learning_rate": 5.212833302556258e-09,
-      "logits/chosen": -2.3403241634368896,
-      "logits/rejected": -2.3502869606018066,
-      "logps/chosen": -411.5088806152344,
-      "logps/rejected": -483.17132568359375,
-      "loss": 0.5333,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -1.0691776275634766,
-      "rewards/margins": 0.5877618789672852,
-      "rewards/rejected": -1.6569395065307617,
+      "epoch": 0.9419152276295133,
+      "grad_norm": 22.341141934616147,
+      "learning_rate": 4.8708793644441086e-09,
+      "logits/chosen": -0.513416588306427,
+      "logits/rejected": -0.3268742263317108,
+      "logps/chosen": -371.934814453125,
+      "logps/rejected": -432.7196350097656,
+      "loss": 0.5312,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.6752878427505493,
+      "rewards/margins": 0.7238127589225769,
+      "rewards/rejected": -1.399100661277771,
       "step": 450
     },
     {
-      "epoch": 0.9623430962343096,
-      "grad_norm": 27.9159850856458,
-      "learning_rate": 2.158697848236607e-09,
-      "logits/chosen": -2.408524990081787,
-      "logits/rejected": -2.397611141204834,
-      "logps/chosen": -388.6316223144531,
-      "logps/rejected": -410.04986572265625,
-      "loss": 0.5385,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -1.0455081462860107,
-      "rewards/margins": 0.5828753709793091,
-      "rewards/rejected": -1.6283836364746094,
+      "epoch": 0.9628466771323915,
+      "grad_norm": 24.32627212356958,
+      "learning_rate": 1.9347820230782295e-09,
+      "logits/chosen": -0.7187199592590332,
+      "logits/rejected": -0.3386891782283783,
+      "logps/chosen": -361.8067932128906,
+      "logps/rejected": -383.07525634765625,
+      "loss": 0.5169,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.6250866651535034,
+      "rewards/margins": 0.7100397348403931,
+      "rewards/rejected": -1.335126280784607,
       "step": 460
     },
     {
-      "epoch": 0.9832635983263598,
-      "grad_norm": 20.82254224019064,
-      "learning_rate": 4.269029751107489e-10,
-      "logits/chosen": -2.3810245990753174,
-      "logits/rejected": -2.3800084590911865,
-      "logps/chosen": -388.88177490234375,
-      "logps/rejected": -439.37091064453125,
-      "loss": 0.5163,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.983181357383728,
-      "rewards/margins": 0.5608035922050476,
-      "rewards/rejected": -1.5439848899841309,
+      "epoch": 0.9837781266352695,
+      "grad_norm": 22.68698991416887,
+      "learning_rate": 3.2839470889836627e-10,
+      "logits/chosen": -0.6799210906028748,
+      "logits/rejected": -0.4490284025669098,
+      "logps/chosen": -387.94158935546875,
+      "logps/rejected": -422.15155029296875,
+      "loss": 0.5164,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.5430302023887634,
+      "rewards/margins": 0.6616371273994446,
+      "rewards/rejected": -1.204667329788208,
       "step": 470
     },
     {
-      "epoch": 1.0,
-      "step": 478,
+      "epoch": 0.9984301412872841,
+      "eval_logits/chosen": -0.44416600465774536,
+      "eval_logits/rejected": -0.26567625999450684,
+      "eval_logps/chosen": -342.55780029296875,
+      "eval_logps/rejected": -420.65606689453125,
+      "eval_loss": 0.533210277557373,
+      "eval_rewards/accuracies": 0.76953125,
+      "eval_rewards/chosen": -0.568053126335144,
+      "eval_rewards/margins": 0.7595610618591309,
+      "eval_rewards/rejected": -1.3276140689849854,
+      "eval_runtime": 169.453,
+      "eval_samples_per_second": 11.803,
+      "eval_steps_per_second": 0.189,
+      "step": 477
+    },
+    {
+      "epoch": 0.9984301412872841,
+      "step": 477,
       "total_flos": 0.0,
-      "train_loss": 0.5718294187569718,
-      "train_runtime": 20140.7322,
-      "train_samples_per_second": 3.035,
-      "train_steps_per_second": 0.024
+      "train_loss": 0.5657099517886244,
+      "train_runtime": 14255.2941,
+      "train_samples_per_second": 4.289,
+      "train_steps_per_second": 0.033
     }
   ],
   "logging_steps": 10,
-  "max_steps": 478,
+  "max_steps": 477,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
-  "save_steps": 100,
+  "save_steps": 500,
   "stateful_callbacks": {
     "TrainerControl": {
       "args": {
@@ -820,7 +772,7 @@
     }
   },
   "total_flos": 0.0,
-  "train_batch_size": 8,
+  "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:aeaadc6a32e3335b706bc8f49fc1c4490d57b4596492200d957418011adb46ac
+oid sha256:acffaabd3239479de4275f5fc33447ecd47659ae9019b4552bdc5d301685b91c
 size 7672