Model save

README.md

@@ -16,15 +16,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model was trained from scratch on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.5136
-- Rewards/chosen: -0.3158
-- Rewards/rejected: -1.2879
-- Rewards/accuracies: 0.7539
-- Rewards/margins: 0.9722
-- Logps/rejected: -289.6701
-- Logps/chosen: -271.4704
-- Logits/rejected: -2.4782
-- Logits/chosen: -2.5194
+- Loss: 0.5007
+- Rewards/chosen: -1.2292
+- Rewards/rejected: -2.2200
+- Rewards/accuracies: 0.7852
+- Rewards/margins: 0.9908
+- Logps/rejected: -477.6671
+- Logps/chosen: -377.2812
+- Logits/rejected: -1.6838
+- Logits/chosen: -1.7495
 
 ## Model description
 
@@ -61,7 +61,7 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.4907        | 0.9984 | 477  | 0.5136          | -0.3158        | -1.2879          | 0.7539             | 0.9722          | -289.6701      | -271.4704    | -2.4782         | -2.5194       |
+| 0.4873        | 0.9984 | 477  | 0.5007          | -1.2292        | -2.2200          | 0.7852             | 0.9908          | -477.6671      | -377.2812    | -1.6838         | -1.7495       |
 
 
 ### Framework versions

all_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.9984301412872841,
     "total_flos": 0.0,
-    "train_loss": 0.5435615640516301,
-    "train_runtime": 13423.4265,
+    "train_loss": 0.5402372428206278,
+    "train_runtime": 15378.7164,
     "train_samples": 61134,
-    "train_samples_per_second": 4.554,
-    "train_steps_per_second": 0.036
+    "train_samples_per_second": 3.975,
+    "train_steps_per_second": 0.031
 }

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "data/zephyr-7b-sft-lr5e-6/checkpoint-500",
+  "_name_or_path": "data/zephyr-7b-sft-lr5e-6/checkpoint-700",
   "architectures": [
     "MistralForCausalLM"
   ],

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1df3a25b5ddc537d871ad1fa3c4d064aea1e6d4d4facdd16c23fd80c005b036b
+oid sha256:978d1b97be53c86f3378a3c78346ee8298582d5f01491f2eb7eca5b82b2e6078
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5c3ac11fcc6ae9aa60d700d6d05af05e5cdc71f08ec66084041dae3c73959580
+oid sha256:fff678ce86a6352a0ffb090caab1ef0bd338fda0560a13fce52bc18b03ca78a4
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:71553997bc1a1cf0cd7d03c296e2460b38840ec8cb78497eb5da9679692926c0
+oid sha256:c9f0652335b95ba85e5d6291b4940aaaec40b98203523ade6f09273805611176
 size 4540516344

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.9984301412872841,
     "total_flos": 0.0,
-    "train_loss": 0.5435615640516301,
-    "train_runtime": 13423.4265,
+    "train_loss": 0.5402372428206278,
+    "train_runtime": 15378.7164,
     "train_samples": 61134,
-    "train_samples_per_second": 4.554,
-    "train_steps_per_second": 0.036
+    "train_samples_per_second": 3.975,
+    "train_steps_per_second": 0.031
 }

trainer_state.json

@@ -10,12 +10,12 @@
   "log_history": [
     {
       "epoch": 0.0020931449502878076,
-      "grad_norm": 56.60030299273906,
+      "grad_norm": 9.34206584891843,
       "learning_rate": 1.0416666666666666e-08,
-      "logits/chosen": -2.8892595767974854,
-      "logits/rejected": -2.840986728668213,
-      "logps/chosen": -312.6982116699219,
-      "logps/rejected": -359.79229736328125,
+      "logits/chosen": -2.6408536434173584,
+      "logits/rejected": -2.5618412494659424,
+      "logps/chosen": -304.3960266113281,
+      "logps/rejected": -351.4190673828125,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -25,733 +25,733 @@
     },
     {
       "epoch": 0.020931449502878074,
-      "grad_norm": 48.0738005946216,
+      "grad_norm": 9.690480435290171,
       "learning_rate": 1.0416666666666667e-07,
-      "logits/chosen": -2.664259672164917,
-      "logits/rejected": -2.628167152404785,
-      "logps/chosen": -319.789306640625,
-      "logps/rejected": -288.3993835449219,
+      "logits/chosen": -2.3715667724609375,
+      "logits/rejected": -2.32132625579834,
+      "logps/chosen": -292.5350646972656,
+      "logps/rejected": -264.379638671875,
       "loss": 0.6931,
-      "rewards/accuracies": 0.4444444477558136,
-      "rewards/chosen": 0.0027708299458026886,
-      "rewards/margins": 0.0005994565435685217,
-      "rewards/rejected": 0.002171373227611184,
+      "rewards/accuracies": 0.4722222089767456,
+      "rewards/chosen": 0.00028340861899778247,
+      "rewards/margins": 0.0003009418142028153,
+      "rewards/rejected": -1.753315700625535e-05,
       "step": 10
     },
     {
       "epoch": 0.04186289900575615,
-      "grad_norm": 67.35204861005953,
+      "grad_norm": 8.704482736945563,
       "learning_rate": 2.0833333333333333e-07,
-      "logits/chosen": -2.743551254272461,
-      "logits/rejected": -2.6782517433166504,
-      "logps/chosen": -310.35101318359375,
-      "logps/rejected": -283.4342956542969,
-      "loss": 0.6913,
-      "rewards/accuracies": 0.5562499761581421,
-      "rewards/chosen": 0.006935287266969681,
-      "rewards/margins": 0.0024963519535958767,
-      "rewards/rejected": 0.004438935313373804,
+      "logits/chosen": -2.47541880607605,
+      "logits/rejected": -2.381457805633545,
+      "logps/chosen": -296.5088806152344,
+      "logps/rejected": -272.64630126953125,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.007291766814887524,
+      "rewards/margins": 0.0026421775110065937,
+      "rewards/rejected": 0.00464958930388093,
       "step": 20
     },
     {
       "epoch": 0.06279434850863422,
-      "grad_norm": 37.91488625769742,
+      "grad_norm": 8.933944908140202,
       "learning_rate": 3.1249999999999997e-07,
-      "logits/chosen": -2.729531764984131,
-      "logits/rejected": -2.6502060890197754,
-      "logps/chosen": -293.98309326171875,
-      "logps/rejected": -251.10726928710938,
-      "loss": 0.6814,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.054005831480026245,
-      "rewards/margins": 0.034085579216480255,
-      "rewards/rejected": 0.019920259714126587,
+      "logits/chosen": -2.4550623893737793,
+      "logits/rejected": -2.3504655361175537,
+      "logps/chosen": -277.6694641113281,
+      "logps/rejected": -237.27053833007812,
+      "loss": 0.6861,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": 0.04149400815367699,
+      "rewards/margins": 0.01431643683463335,
+      "rewards/rejected": 0.02717757597565651,
       "step": 30
     },
     {
       "epoch": 0.0837257980115123,
-      "grad_norm": 36.97741406624794,
+      "grad_norm": 9.199681453566614,
       "learning_rate": 4.1666666666666667e-07,
-      "logits/chosen": -2.625835657119751,
-      "logits/rejected": -2.5816197395324707,
-      "logps/chosen": -273.37054443359375,
-      "logps/rejected": -261.5669250488281,
-      "loss": 0.6582,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.12765000760555267,
-      "rewards/margins": 0.08208973705768585,
-      "rewards/rejected": 0.04556027799844742,
+      "logits/chosen": -2.3616414070129395,
+      "logits/rejected": -2.3001840114593506,
+      "logps/chosen": -250.1017608642578,
+      "logps/rejected": -252.5328826904297,
+      "loss": 0.6716,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.05417194217443466,
+      "rewards/margins": 0.048414021730422974,
+      "rewards/rejected": 0.0057579209096729755,
       "step": 40
     },
     {
       "epoch": 0.10465724751439037,
-      "grad_norm": 37.43059499290312,
+      "grad_norm": 10.235470501727459,
       "learning_rate": 4.999731868769026e-07,
-      "logits/chosen": -2.6400301456451416,
-      "logits/rejected": -2.5608487129211426,
-      "logps/chosen": -275.17523193359375,
-      "logps/rejected": -275.0105285644531,
-      "loss": 0.6272,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": 0.20204421877861023,
-      "rewards/margins": 0.16956469416618347,
-      "rewards/rejected": 0.03247954696416855,
+      "logits/chosen": -2.4129481315612793,
+      "logits/rejected": -2.3142497539520264,
+      "logps/chosen": -268.14239501953125,
+      "logps/rejected": -273.57891845703125,
+      "loss": 0.6474,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.001057840185239911,
+      "rewards/margins": 0.10867471992969513,
+      "rewards/rejected": -0.10973255336284637,
       "step": 50
     },
     {
       "epoch": 0.12558869701726844,
-      "grad_norm": 57.10845065545151,
+      "grad_norm": 19.354984786530007,
       "learning_rate": 4.990353313429303e-07,
-      "logits/chosen": -2.642455577850342,
-      "logits/rejected": -2.573876142501831,
-      "logps/chosen": -251.33602905273438,
-      "logps/rejected": -249.27578735351562,
-      "loss": 0.5981,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.23599937558174133,
-      "rewards/margins": 0.3313826322555542,
-      "rewards/rejected": -0.09538328647613525,
+      "logits/chosen": -2.435051441192627,
+      "logits/rejected": -2.355966567993164,
+      "logps/chosen": -254.5166778564453,
+      "logps/rejected": -267.9598388671875,
+      "loss": 0.6136,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.09817924350500107,
+      "rewards/margins": 0.21375682950019836,
+      "rewards/rejected": -0.31193608045578003,
       "step": 60
     },
     {
       "epoch": 0.14652014652014653,
-      "grad_norm": 35.22629948353397,
+      "grad_norm": 15.93843826908321,
       "learning_rate": 4.967625656594781e-07,
-      "logits/chosen": -2.5455336570739746,
-      "logits/rejected": -2.5049350261688232,
-      "logps/chosen": -302.450927734375,
-      "logps/rejected": -292.55072021484375,
-      "loss": 0.5917,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": 0.12116215378046036,
-      "rewards/margins": 0.40936508774757385,
-      "rewards/rejected": -0.2882029414176941,
+      "logits/chosen": -2.3557989597320557,
+      "logits/rejected": -2.3143229484558105,
+      "logps/chosen": -307.20587158203125,
+      "logps/rejected": -323.688232421875,
+      "loss": 0.5986,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.22956323623657227,
+      "rewards/margins": 0.2852025628089905,
+      "rewards/rejected": -0.5147657990455627,
       "step": 70
     },
     {
       "epoch": 0.1674515960230246,
-      "grad_norm": 47.22877045853146,
+      "grad_norm": 17.008007053555076,
       "learning_rate": 4.93167072587771e-07,
-      "logits/chosen": -2.6536507606506348,
-      "logits/rejected": -2.5224609375,
-      "logps/chosen": -340.2115783691406,
-      "logps/rejected": -274.326904296875,
-      "loss": 0.5907,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": 0.13078466057777405,
-      "rewards/margins": 0.5240527391433716,
-      "rewards/rejected": -0.39326804876327515,
+      "logits/chosen": -2.508552074432373,
+      "logits/rejected": -2.3846137523651123,
+      "logps/chosen": -353.18878173828125,
+      "logps/rejected": -323.2257385253906,
+      "loss": 0.5958,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.3706515431404114,
+      "rewards/margins": 0.3792355954647064,
+      "rewards/rejected": -0.7498871684074402,
       "step": 80
     },
     {
       "epoch": 0.18838304552590268,
-      "grad_norm": 37.16410104338103,
+      "grad_norm": 21.842890244155406,
       "learning_rate": 4.882681251368548e-07,
-      "logits/chosen": -2.5428051948547363,
-      "logits/rejected": -2.4835638999938965,
-      "logps/chosen": -254.80374145507812,
-      "logps/rejected": -269.6417236328125,
-      "loss": 0.5481,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.027083083987236023,
-      "rewards/margins": 0.6100192070007324,
-      "rewards/rejected": -0.6371022462844849,
+      "logits/chosen": -2.3856592178344727,
+      "logits/rejected": -2.324418544769287,
+      "logps/chosen": -304.6998596191406,
+      "logps/rejected": -354.70465087890625,
+      "loss": 0.5632,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.6071581840515137,
+      "rewards/margins": 0.4484094977378845,
+      "rewards/rejected": -1.055567741394043,
       "step": 90
     },
     {
       "epoch": 0.20931449502878074,
-      "grad_norm": 33.7175693505583,
+      "grad_norm": 16.240227508625306,
       "learning_rate": 4.820919832540181e-07,
-      "logits/chosen": -2.525334119796753,
-      "logits/rejected": -2.4599850177764893,
-      "logps/chosen": -305.0451965332031,
-      "logps/rejected": -294.32122802734375,
-      "loss": 0.5775,
+      "logits/chosen": -2.3550620079040527,
+      "logits/rejected": -2.287639856338501,
+      "logps/chosen": -336.8813171386719,
+      "logps/rejected": -371.5824890136719,
+      "loss": 0.5708,
       "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.04258927330374718,
-      "rewards/margins": 0.7190525531768799,
-      "rewards/rejected": -0.7616418600082397,
+      "rewards/chosen": -0.5176103711128235,
+      "rewards/margins": 0.5057469606399536,
+      "rewards/rejected": -1.0233572721481323,
       "step": 100
     },
     {
       "epoch": 0.2302459445316588,
-      "grad_norm": 41.20672223986865,
+      "grad_norm": 21.75942595967658,
       "learning_rate": 4.7467175306295647e-07,
-      "logits/chosen": -2.5379371643066406,
-      "logits/rejected": -2.4624836444854736,
-      "logps/chosen": -295.3846130371094,
-      "logps/rejected": -290.00823974609375,
-      "loss": 0.5683,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.09885083138942719,
-      "rewards/margins": 0.5707240700721741,
-      "rewards/rejected": -0.6695749163627625,
+      "logits/chosen": -2.3279662132263184,
+      "logits/rejected": -2.2442610263824463,
+      "logps/chosen": -343.7782287597656,
+      "logps/rejected": -372.44189453125,
+      "loss": 0.5781,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.6124948263168335,
+      "rewards/margins": 0.4476219117641449,
+      "rewards/rejected": -1.0601167678833008,
       "step": 110
     },
     {
       "epoch": 0.25117739403453687,
-      "grad_norm": 38.75944481368775,
+      "grad_norm": 18.92465346221188,
       "learning_rate": 4.6604720940421207e-07,
-      "logits/chosen": -2.4620394706726074,
-      "logits/rejected": -2.453782558441162,
-      "logps/chosen": -280.3204040527344,
-      "logps/rejected": -284.44219970703125,
-      "loss": 0.5627,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.15607118606567383,
-      "rewards/margins": 0.6097468137741089,
-      "rewards/rejected": -0.7658179998397827,
+      "logits/chosen": -2.1767220497131348,
+      "logits/rejected": -2.161332368850708,
+      "logps/chosen": -321.2099914550781,
+      "logps/rejected": -374.1365051269531,
+      "loss": 0.5556,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.624541163444519,
+      "rewards/margins": 0.5819947719573975,
+      "rewards/rejected": -1.2065359354019165,
       "step": 120
     },
     {
       "epoch": 0.272108843537415,
-      "grad_norm": 58.76765744852648,
+      "grad_norm": 17.122180048395105,
       "learning_rate": 4.5626458262912735e-07,
-      "logits/chosen": -2.52392315864563,
-      "logits/rejected": -2.481851100921631,
-      "logps/chosen": -299.79156494140625,
-      "logps/rejected": -292.2509765625,
-      "loss": 0.5275,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.26476210355758667,
-      "rewards/margins": 0.6276587843894958,
-      "rewards/rejected": -0.8924208879470825,
+      "logits/chosen": -2.221346378326416,
+      "logits/rejected": -2.1790013313293457,
+      "logps/chosen": -353.6302795410156,
+      "logps/rejected": -388.6443786621094,
+      "loss": 0.5391,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.825293242931366,
+      "rewards/margins": 0.4637347161769867,
+      "rewards/rejected": -1.2890280485153198,
       "step": 130
     },
     {
       "epoch": 0.29304029304029305,
-      "grad_norm": 64.42453786695027,
+      "grad_norm": 17.095632859067344,
       "learning_rate": 4.453763107901675e-07,
-      "logits/chosen": -2.546255588531494,
-      "logits/rejected": -2.483006715774536,
-      "logps/chosen": -318.06182861328125,
-      "logps/rejected": -286.57257080078125,
-      "loss": 0.5517,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": 0.06344182789325714,
-      "rewards/margins": 0.8681875467300415,
-      "rewards/rejected": -0.8047456741333008,
+      "logits/chosen": -2.158041477203369,
+      "logits/rejected": -2.0732905864715576,
+      "logps/chosen": -355.7929382324219,
+      "logps/rejected": -372.53021240234375,
+      "loss": 0.552,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.5128291845321655,
+      "rewards/margins": 0.5925513505935669,
+      "rewards/rejected": -1.1053804159164429,
       "step": 140
     },
     {
       "epoch": 0.3139717425431711,
-      "grad_norm": 49.33320813589767,
+      "grad_norm": 24.657788255904606,
       "learning_rate": 4.3344075855595097e-07,
-      "logits/chosen": -2.5776188373565674,
-      "logits/rejected": -2.5224177837371826,
-      "logps/chosen": -294.88134765625,
-      "logps/rejected": -271.59539794921875,
-      "loss": 0.5302,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": 0.04772792011499405,
-      "rewards/margins": 0.6878777742385864,
-      "rewards/rejected": -0.6401498317718506,
+      "logits/chosen": -2.0279147624969482,
+      "logits/rejected": -1.9383026361465454,
+      "logps/chosen": -328.5897521972656,
+      "logps/rejected": -343.27105712890625,
+      "loss": 0.5536,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.582419216632843,
+      "rewards/margins": 0.45015817880630493,
+      "rewards/rejected": -1.0325775146484375,
       "step": 150
     },
     {
       "epoch": 0.3349031920460492,
-      "grad_norm": 41.399344035646344,
+      "grad_norm": 20.8468757303537,
       "learning_rate": 4.2052190435769554e-07,
-      "logits/chosen": -2.6206777095794678,
-      "logits/rejected": -2.5290703773498535,
-      "logps/chosen": -279.5448303222656,
-      "logps/rejected": -271.38916015625,
-      "loss": 0.5468,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.03517674654722214,
-      "rewards/margins": 0.8822032809257507,
-      "rewards/rejected": -0.9173799753189087,
+      "logits/chosen": -2.1219663619995117,
+      "logits/rejected": -1.9995368719100952,
+      "logps/chosen": -340.55560302734375,
+      "logps/rejected": -381.2513732910156,
+      "loss": 0.5544,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.7368024587631226,
+      "rewards/margins": 0.6101049780845642,
+      "rewards/rejected": -1.346907377243042,
       "step": 160
     },
     {
       "epoch": 0.35583464154892724,
-      "grad_norm": 39.72842833528202,
+      "grad_norm": 25.75784518184545,
       "learning_rate": 4.0668899744407567e-07,
-      "logits/chosen": -2.4758784770965576,
-      "logits/rejected": -2.437650442123413,
-      "logps/chosen": -268.2413330078125,
-      "logps/rejected": -257.02105712890625,
-      "loss": 0.5403,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.12392053753137589,
-      "rewards/margins": 0.5926230549812317,
-      "rewards/rejected": -0.7165434956550598,
+      "logits/chosen": -2.0343892574310303,
+      "logits/rejected": -1.9912550449371338,
+      "logps/chosen": -333.8672180175781,
+      "logps/rejected": -364.148681640625,
+      "loss": 0.5411,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.8337874412536621,
+      "rewards/margins": 0.515734076499939,
+      "rewards/rejected": -1.349521517753601,
       "step": 170
     },
     {
       "epoch": 0.37676609105180536,
-      "grad_norm": 36.98022789966047,
+      "grad_norm": 22.431217860948216,
       "learning_rate": 3.920161866827889e-07,
-      "logits/chosen": -2.5508599281311035,
-      "logits/rejected": -2.50437593460083,
-      "logps/chosen": -284.5989074707031,
-      "logps/rejected": -273.63800048828125,
-      "loss": 0.5216,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.47948211431503296,
-      "rewards/margins": 0.6584511995315552,
-      "rewards/rejected": -1.137933373451233,
+      "logits/chosen": -2.160536289215088,
+      "logits/rejected": -2.108321189880371,
+      "logps/chosen": -356.4222412109375,
+      "logps/rejected": -384.0712585449219,
+      "loss": 0.5158,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.9323843121528625,
+      "rewards/margins": 0.526262640953064,
+      "rewards/rejected": -1.4586468935012817,
       "step": 180
     },
     {
       "epoch": 0.3976975405546834,
-      "grad_norm": 83.96111613389628,
+      "grad_norm": 23.80320167821421,
       "learning_rate": 3.765821230985757e-07,
-      "logits/chosen": -2.479034185409546,
-      "logits/rejected": -2.459357261657715,
-      "logps/chosen": -265.3419189453125,
-      "logps/rejected": -273.99237060546875,
-      "loss": 0.5357,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.2723972201347351,
-      "rewards/margins": 0.5891710519790649,
-      "rewards/rejected": -0.8615682721138,
+      "logits/chosen": -2.111511707305908,
+      "logits/rejected": -2.0807371139526367,
+      "logps/chosen": -330.0537109375,
+      "logps/rejected": -375.9040222167969,
+      "loss": 0.5398,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.7908061742782593,
+      "rewards/margins": 0.5114402770996094,
+      "rewards/rejected": -1.3022464513778687,
       "step": 190
     },
     {
       "epoch": 0.4186289900575615,
-      "grad_norm": 57.98472670273764,
+      "grad_norm": 46.83083982513084,
       "learning_rate": 3.604695382782159e-07,
-      "logits/chosen": -2.4622385501861572,
-      "logits/rejected": -2.453322172164917,
-      "logps/chosen": -261.9639587402344,
-      "logps/rejected": -290.62188720703125,
-      "loss": 0.5961,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.21924467384815216,
-      "rewards/margins": 0.6115727424621582,
-      "rewards/rejected": -0.8308174014091492,
+      "logits/chosen": -2.1599907875061035,
+      "logits/rejected": -2.159285068511963,
+      "logps/chosen": -325.5718688964844,
+      "logps/rejected": -401.55718994140625,
+      "loss": 0.5406,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.8464052081108093,
+      "rewards/margins": 0.5457392334938049,
+      "rewards/rejected": -1.3921444416046143,
       "step": 200
     },
     {
       "epoch": 0.43956043956043955,
-      "grad_norm": 58.79891040438903,
+      "grad_norm": 30.809983774424474,
       "learning_rate": 3.4376480090239047e-07,
-      "logits/chosen": -2.5712344646453857,
-      "logits/rejected": -2.463402509689331,
-      "logps/chosen": -318.231201171875,
-      "logps/rejected": -269.74798583984375,
-      "loss": 0.5426,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.1611379086971283,
-      "rewards/margins": 0.7542437314987183,
-      "rewards/rejected": -0.9153816103935242,
+      "logits/chosen": -2.1967430114746094,
+      "logits/rejected": -2.0988197326660156,
+      "logps/chosen": -386.5250244140625,
+      "logps/rejected": -397.96429443359375,
+      "loss": 0.5319,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.8952194452285767,
+      "rewards/margins": 0.643934428691864,
+      "rewards/rejected": -1.539153814315796,
       "step": 210
     },
     {
       "epoch": 0.4604918890633176,
-      "grad_norm": 79.8856418955716,
+      "grad_norm": 36.142986008936305,
       "learning_rate": 3.265574537815398e-07,
-      "logits/chosen": -2.5093963146209717,
-      "logits/rejected": -2.497286319732666,
-      "logps/chosen": -270.76568603515625,
-      "logps/rejected": -272.580078125,
-      "loss": 0.5446,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.10513055324554443,
-      "rewards/margins": 0.9609693288803101,
-      "rewards/rejected": -1.0660998821258545,
+      "logits/chosen": -1.9691970348358154,
+      "logits/rejected": -1.966980218887329,
+      "logps/chosen": -332.0834655761719,
+      "logps/rejected": -407.1396789550781,
+      "loss": 0.5245,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.0065058469772339,
+      "rewards/margins": 0.709026575088501,
+      "rewards/rejected": -1.7155323028564453,
       "step": 220
     },
     {
       "epoch": 0.48142333856619574,
-      "grad_norm": 38.74064389181431,
+      "grad_norm": 32.105309802311105,
       "learning_rate": 3.0893973387735683e-07,
-      "logits/chosen": -2.6367149353027344,
-      "logits/rejected": -2.564960479736328,
-      "logps/chosen": -273.73529052734375,
-      "logps/rejected": -270.94415283203125,
-      "loss": 0.5278,
+      "logits/chosen": -2.0395565032958984,
+      "logits/rejected": -1.947618842124939,
+      "logps/chosen": -370.189208984375,
+      "logps/rejected": -454.5423889160156,
+      "loss": 0.5372,
       "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -0.16755743324756622,
-      "rewards/margins": 1.0889314413070679,
-      "rewards/rejected": -1.2564888000488281,
+      "rewards/chosen": -1.1879613399505615,
+      "rewards/margins": 1.0171879529953003,
+      "rewards/rejected": -2.2051491737365723,
       "step": 230
     },
     {
       "epoch": 0.5023547880690737,
-      "grad_norm": 40.4843251117903,
+      "grad_norm": 28.941311573129713,
       "learning_rate": 2.910060778827554e-07,
-      "logits/chosen": -2.633864641189575,
-      "logits/rejected": -2.5612056255340576,
-      "logps/chosen": -296.2731628417969,
-      "logps/rejected": -290.04742431640625,
-      "loss": 0.4978,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.12019411474466324,
-      "rewards/margins": 0.7597166895866394,
-      "rewards/rejected": -0.8799108266830444,
+      "logits/chosen": -2.0586354732513428,
+      "logits/rejected": -1.9693540334701538,
+      "logps/chosen": -394.671142578125,
+      "logps/rejected": -445.7662658691406,
+      "loss": 0.4933,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.167217493057251,
+      "rewards/margins": 0.7689393758773804,
+      "rewards/rejected": -1.9361568689346313,
       "step": 240
     },
     {
       "epoch": 0.5232862375719518,
-      "grad_norm": 40.628977686293446,
+      "grad_norm": 32.453932975157784,
       "learning_rate": 2.7285261601056697e-07,
-      "logits/chosen": -2.611175060272217,
-      "logits/rejected": -2.517091989517212,
-      "logps/chosen": -294.7513122558594,
-      "logps/rejected": -285.98199462890625,
-      "loss": 0.5307,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -0.2056296169757843,
-      "rewards/margins": 1.1986558437347412,
-      "rewards/rejected": -1.4042854309082031,
+      "logits/chosen": -2.033799886703491,
+      "logits/rejected": -1.9187284708023071,
+      "logps/chosen": -374.45587158203125,
+      "logps/rejected": -424.2225646972656,
+      "loss": 0.5141,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.0642242431640625,
+      "rewards/margins": 0.867928683757782,
+      "rewards/rejected": -1.9321529865264893,
       "step": 250
     },
     {
       "epoch": 0.54421768707483,
-      "grad_norm": 41.00335841137649,
+      "grad_norm": 38.54774504177246,
       "learning_rate": 2.5457665670441937e-07,
-      "logits/chosen": -2.6146559715270996,
-      "logits/rejected": -2.608934164047241,
-      "logps/chosen": -274.4793395996094,
-      "logps/rejected": -270.13726806640625,
-      "loss": 0.5159,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.43834853172302246,
-      "rewards/margins": 0.8028098344802856,
-      "rewards/rejected": -1.241158366203308,
+      "logits/chosen": -2.022801160812378,
+      "logits/rejected": -2.0115959644317627,
+      "logps/chosen": -364.87188720703125,
+      "logps/rejected": -422.316650390625,
+      "loss": 0.5095,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.0906543731689453,
+      "rewards/margins": 0.7382703423500061,
+      "rewards/rejected": -1.8289247751235962,
       "step": 260
     },
     {
       "epoch": 0.565149136577708,
-      "grad_norm": 38.419381160509054,
+      "grad_norm": 38.637611601586094,
       "learning_rate": 2.3627616503391812e-07,
-      "logits/chosen": -2.6012203693389893,
-      "logits/rejected": -2.5700631141662598,
-      "logps/chosen": -302.5823059082031,
-      "logps/rejected": -297.1186218261719,
-      "loss": 0.5141,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.3534887433052063,
-      "rewards/margins": 1.0269334316253662,
-      "rewards/rejected": -1.3804222345352173,
+      "logits/chosen": -1.961382269859314,
+      "logits/rejected": -1.9140493869781494,
+      "logps/chosen": -412.92926025390625,
+      "logps/rejected": -487.8951110839844,
+      "loss": 0.5254,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.3046131134033203,
+      "rewards/margins": 0.9779068231582642,
+      "rewards/rejected": -2.282519817352295,
       "step": 270
     },
     {
       "epoch": 0.5860805860805861,
-      "grad_norm": 61.56800465863709,
+      "grad_norm": 38.46041638764684,
       "learning_rate": 2.1804923757009882e-07,
-      "logits/chosen": -2.487819194793701,
-      "logits/rejected": -2.443556308746338,
-      "logps/chosen": -289.46136474609375,
-      "logps/rejected": -258.09454345703125,
-      "loss": 0.5389,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.3526845872402191,
-      "rewards/margins": 0.8081240653991699,
-      "rewards/rejected": -1.1608086824417114,
+      "logits/chosen": -1.8148670196533203,
+      "logits/rejected": -1.759767770767212,
+      "logps/chosen": -390.0211486816406,
+      "logps/rejected": -435.19921875,
+      "loss": 0.5251,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.3935781717300415,
+      "rewards/margins": 0.6986350417137146,
+      "rewards/rejected": -2.0922131538391113,
       "step": 280
     },
     {
       "epoch": 0.6070120355834642,
-      "grad_norm": 42.020314742546624,
+      "grad_norm": 28.17757740457026,
       "learning_rate": 1.9999357655598891e-07,
-      "logits/chosen": -2.587932586669922,
-      "logits/rejected": -2.517599582672119,
-      "logps/chosen": -262.06488037109375,
-      "logps/rejected": -273.3334045410156,
-      "loss": 0.536,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -0.5786847472190857,
-      "rewards/margins": 0.6844012141227722,
-      "rewards/rejected": -1.2630858421325684,
+      "logits/chosen": -1.9572741985321045,
+      "logits/rejected": -1.8640228509902954,
+      "logps/chosen": -362.7040100097656,
+      "logps/rejected": -433.5982360839844,
+      "loss": 0.5155,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.2248718738555908,
+      "rewards/margins": 0.7737196683883667,
+      "rewards/rejected": -1.998591423034668,
       "step": 290
     },
     {
       "epoch": 0.6279434850863422,
-      "grad_norm": 45.77982161189827,
+      "grad_norm": 26.314213240746867,
       "learning_rate": 1.8220596619089573e-07,
-      "logits/chosen": -2.551849603652954,
-      "logits/rejected": -2.4827933311462402,
-      "logps/chosen": -338.84259033203125,
-      "logps/rejected": -313.6923828125,
-      "loss": 0.5116,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.437159925699234,
-      "rewards/margins": 0.7780275940895081,
-      "rewards/rejected": -1.215187430381775,
+      "logits/chosen": -1.938108205795288,
+      "logits/rejected": -1.8487415313720703,
+      "logps/chosen": -410.2854919433594,
+      "logps/rejected": -449.3778381347656,
+      "loss": 0.4933,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.025935411453247,
+      "rewards/margins": 0.8152927160263062,
+      "rewards/rejected": -1.8412281274795532,
       "step": 300
     },
     {
       "epoch": 0.6488749345892203,
-      "grad_norm": 67.26009798996141,
+      "grad_norm": 29.022869814488782,
       "learning_rate": 1.647817538357072e-07,
-      "logits/chosen": -2.586299419403076,
-      "logits/rejected": -2.5306968688964844,
-      "logps/chosen": -315.3221130371094,
-      "logps/rejected": -284.2604064941406,
-      "loss": 0.5072,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.25200536847114563,
-      "rewards/margins": 0.9943742752075195,
-      "rewards/rejected": -1.2463796138763428,
+      "logits/chosen": -1.9548978805541992,
+      "logits/rejected": -1.879836082458496,
+      "logps/chosen": -414.93963623046875,
+      "logps/rejected": -457.4412536621094,
+      "loss": 0.4966,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.235777735710144,
+      "rewards/margins": 0.9157119989395142,
+      "rewards/rejected": -2.151489734649658,
       "step": 310
     },
     {
       "epoch": 0.6698063840920984,
-      "grad_norm": 41.456508803565676,
+      "grad_norm": 30.108541803739723,
       "learning_rate": 1.478143389201113e-07,
-      "logits/chosen": -2.5629897117614746,
-      "logits/rejected": -2.4874186515808105,
-      "logps/chosen": -265.6336669921875,
-      "logps/rejected": -254.78158569335938,
-      "loss": 0.5157,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -0.22662892937660217,
-      "rewards/margins": 1.0545392036437988,
-      "rewards/rejected": -1.2811682224273682,
+      "logits/chosen": -1.8911720514297485,
+      "logits/rejected": -1.7862389087677002,
+      "logps/chosen": -366.5466003417969,
+      "logps/rejected": -432.31427001953125,
+      "loss": 0.4868,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -1.155328631401062,
+      "rewards/margins": 0.9571216702461243,
+      "rewards/rejected": -2.112450361251831,
       "step": 320
     },
     {
       "epoch": 0.6907378335949764,
-      "grad_norm": 36.05897194603855,
+      "grad_norm": 28.000171872717303,
       "learning_rate": 1.3139467229135998e-07,
-      "logits/chosen": -2.5867385864257812,
-      "logits/rejected": -2.570345878601074,
-      "logps/chosen": -279.125,
-      "logps/rejected": -296.114501953125,
-      "loss": 0.5157,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.22260840237140656,
-      "rewards/margins": 0.8764309883117676,
-      "rewards/rejected": -1.0990393161773682,
+      "logits/chosen": -1.917880654335022,
+      "logits/rejected": -1.8926366567611694,
+      "logps/chosen": -363.8395080566406,
+      "logps/rejected": -440.37548828125,
+      "loss": 0.5037,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.0377060174942017,
+      "rewards/margins": 0.7931325435638428,
+      "rewards/rejected": -1.8308385610580444,
       "step": 330
     },
     {
       "epoch": 0.7116692830978545,
-      "grad_norm": 48.915795401449174,
+      "grad_norm": 34.53057751025992,
       "learning_rate": 1.1561076868822755e-07,
-      "logits/chosen": -2.541907548904419,
-      "logits/rejected": -2.506343126296997,
-      "logps/chosen": -307.07427978515625,
-      "logps/rejected": -305.878662109375,
-      "loss": 0.4979,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.31391283869743347,
-      "rewards/margins": 0.9292833209037781,
-      "rewards/rejected": -1.2431962490081787,
+      "logits/chosen": -1.8293085098266602,
+      "logits/rejected": -1.7758848667144775,
+      "logps/chosen": -400.27154541015625,
+      "logps/rejected": -463.6654357910156,
+      "loss": 0.5004,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.2170968055725098,
+      "rewards/margins": 0.8406441807746887,
+      "rewards/rejected": -2.057741165161133,
       "step": 340
     },
     {
       "epoch": 0.7326007326007326,
-      "grad_norm": 42.37289903395956,
+      "grad_norm": 28.73749633774597,
       "learning_rate": 1.0054723495346482e-07,
-      "logits/chosen": -2.6136066913604736,
-      "logits/rejected": -2.54303240776062,
-      "logps/chosen": -274.64434814453125,
-      "logps/rejected": -279.75482177734375,
-      "loss": 0.4802,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -0.19893045723438263,
-      "rewards/margins": 0.978558361530304,
-      "rewards/rejected": -1.1774885654449463,
+      "logits/chosen": -1.8887624740600586,
+      "logits/rejected": -1.7918049097061157,
+      "logps/chosen": -376.71734619140625,
+      "logps/rejected": -443.11822509765625,
+      "loss": 0.4659,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -1.2517045736312866,
+      "rewards/margins": 0.9187850952148438,
+      "rewards/rejected": -2.17048978805542,
       "step": 350
     },
     {
       "epoch": 0.7535321821036107,
-      "grad_norm": 38.677534171799124,
+      "grad_norm": 27.32892904184351,
       "learning_rate": 8.628481651367875e-08,
-      "logits/chosen": -2.58419132232666,
-      "logits/rejected": -2.518683910369873,
-      "logps/chosen": -319.4009094238281,
-      "logps/rejected": -292.9573974609375,
-      "loss": 0.5281,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.31927159428596497,
-      "rewards/margins": 1.0226125717163086,
-      "rewards/rejected": -1.3418842554092407,
+      "logits/chosen": -1.8553106784820557,
+      "logits/rejected": -1.7398430109024048,
+      "logps/chosen": -430.87066650390625,
+      "logps/rejected": -485.1336364746094,
+      "loss": 0.5152,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -1.3292081356048584,
+      "rewards/margins": 0.9499826431274414,
+      "rewards/rejected": -2.279191017150879,
       "step": 360
     },
     {
       "epoch": 0.7744636316064888,
-      "grad_norm": 46.36575225037569,
+      "grad_norm": 29.199443383027504,
       "learning_rate": 7.289996455765748e-08,
-      "logits/chosen": -2.5846099853515625,
-      "logits/rejected": -2.5254452228546143,
-      "logps/chosen": -284.2174072265625,
-      "logps/rejected": -264.4429626464844,
-      "loss": 0.5244,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.36106812953948975,
-      "rewards/margins": 0.966556191444397,
-      "rewards/rejected": -1.3276244401931763,
+      "logits/chosen": -1.8290824890136719,
+      "logits/rejected": -1.7347335815429688,
+      "logps/chosen": -394.8559265136719,
+      "logps/rejected": -456.19488525390625,
+      "loss": 0.5079,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.356519103050232,
+      "rewards/margins": 0.8976017236709595,
+      "rewards/rejected": -2.2541210651397705,
       "step": 370
     },
     {
       "epoch": 0.7953950811093669,
-      "grad_norm": 39.86634863792299,
+      "grad_norm": 27.421591442142596,
       "learning_rate": 6.046442623320145e-08,
-      "logits/chosen": -2.47021484375,
-      "logits/rejected": -2.45784854888916,
-      "logps/chosen": -276.29632568359375,
-      "logps/rejected": -316.617431640625,
-      "loss": 0.5091,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.7333633303642273,
-      "rewards/margins": 1.0854023694992065,
-      "rewards/rejected": -1.8187658786773682,
+      "logits/chosen": -1.6712805032730103,
+      "logits/rejected": -1.6388919353485107,
+      "logps/chosen": -399.28936767578125,
+      "logps/rejected": -523.0585327148438,
+      "loss": 0.5035,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.6077839136123657,
+      "rewards/margins": 1.0587600469589233,
+      "rewards/rejected": -2.666543960571289,
       "step": 380
     },
     {
       "epoch": 0.8163265306122449,
-      "grad_norm": 40.36648943435987,
+      "grad_norm": 30.359012124716116,
       "learning_rate": 4.904486005914027e-08,
-      "logits/chosen": -2.5730109214782715,
-      "logits/rejected": -2.515702962875366,
-      "logps/chosen": -364.7529296875,
-      "logps/rejected": -344.87060546875,
-      "loss": 0.4929,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.5375163555145264,
-      "rewards/margins": 1.0361560583114624,
-      "rewards/rejected": -1.5736725330352783,
+      "logits/chosen": -1.8206322193145752,
+      "logits/rejected": -1.7272882461547852,
+      "logps/chosen": -474.2603454589844,
+      "logps/rejected": -522.3705444335938,
+      "loss": 0.4918,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.457648515701294,
+      "rewards/margins": 0.8391188383102417,
+      "rewards/rejected": -2.296767473220825,
       "step": 390
     },
     {
       "epoch": 0.837257980115123,
-      "grad_norm": 56.32657464205283,
+      "grad_norm": 28.94974585374415,
       "learning_rate": 3.8702478614051345e-08,
-      "logits/chosen": -2.525852918624878,
-      "logits/rejected": -2.469883680343628,
-      "logps/chosen": -276.40948486328125,
-      "logps/rejected": -274.75274658203125,
-      "loss": 0.5189,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.5351244211196899,
-      "rewards/margins": 0.9622423052787781,
-      "rewards/rejected": -1.4973666667938232,
+      "logits/chosen": -1.7498900890350342,
+      "logits/rejected": -1.664999008178711,
+      "logps/chosen": -377.47552490234375,
+      "logps/rejected": -452.85772705078125,
+      "loss": 0.5108,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.2954479455947876,
+      "rewards/margins": 0.8770959973335266,
+      "rewards/rejected": -2.172544002532959,
       "step": 400
     },
     {
       "epoch": 0.858189429618001,
-      "grad_norm": 42.87274307233854,
+      "grad_norm": 28.935243126672802,
       "learning_rate": 2.9492720416985e-08,
-      "logits/chosen": -2.6081812381744385,
-      "logits/rejected": -2.5553929805755615,
-      "logps/chosen": -317.64312744140625,
-      "logps/rejected": -301.38323974609375,
-      "loss": 0.5085,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.42447155714035034,
-      "rewards/margins": 1.0819838047027588,
-      "rewards/rejected": -1.506455421447754,
+      "logits/chosen": -1.8779255151748657,
+      "logits/rejected": -1.786258339881897,
+      "logps/chosen": -422.50775146484375,
+      "logps/rejected": -483.96392822265625,
+      "loss": 0.5096,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.3002262115478516,
+      "rewards/margins": 0.9358274340629578,
+      "rewards/rejected": -2.236053705215454,
       "step": 410
     },
     {
       "epoch": 0.8791208791208791,
-      "grad_norm": 41.8015801763105,
+      "grad_norm": 29.945810967167397,
       "learning_rate": 2.1464952759020856e-08,
-      "logits/chosen": -2.502769708633423,
-      "logits/rejected": -2.475386381149292,
-      "logps/chosen": -273.35150146484375,
-      "logps/rejected": -296.51361083984375,
-      "loss": 0.4965,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.42012089490890503,
-      "rewards/margins": 0.9804800152778625,
-      "rewards/rejected": -1.4006009101867676,
+      "logits/chosen": -1.7313286066055298,
+      "logits/rejected": -1.6857627630233765,
+      "logps/chosen": -381.6114807128906,
+      "logps/rejected": -479.25738525390625,
+      "loss": 0.4962,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.3308442831039429,
+      "rewards/margins": 0.9352275729179382,
+      "rewards/rejected": -2.2660717964172363,
       "step": 420
     },
     {
       "epoch": 0.9000523286237572,
-      "grad_norm": 42.916394522460244,
+      "grad_norm": 26.119384851501806,
       "learning_rate": 1.4662207078575684e-08,
-      "logits/chosen": -2.516355037689209,
-      "logits/rejected": -2.4368109703063965,
-      "logps/chosen": -307.8439025878906,
-      "logps/rejected": -306.0263671875,
-      "loss": 0.4975,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.3919011950492859,
-      "rewards/margins": 0.8122466802597046,
-      "rewards/rejected": -1.2041478157043457,
+      "logits/chosen": -1.7514724731445312,
+      "logits/rejected": -1.6334092617034912,
+      "logps/chosen": -412.2776794433594,
+      "logps/rejected": -485.67474365234375,
+      "loss": 0.4761,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -1.3133318424224854,
+      "rewards/margins": 0.9652267694473267,
+      "rewards/rejected": -2.2785582542419434,
       "step": 430
     },
     {
       "epoch": 0.9209837781266352,
-      "grad_norm": 39.402077703083755,
+      "grad_norm": 26.94588013879126,
       "learning_rate": 9.12094829893642e-09,
-      "logits/chosen": -2.6104936599731445,
-      "logits/rejected": -2.5414485931396484,
-      "logps/chosen": -272.779052734375,
-      "logps/rejected": -253.19808959960938,
-      "loss": 0.5093,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.33945396542549133,
-      "rewards/margins": 0.873080849647522,
-      "rewards/rejected": -1.212534785270691,
+      "logits/chosen": -1.8784576654434204,
+      "logits/rejected": -1.7693846225738525,
+      "logps/chosen": -380.4332275390625,
+      "logps/rejected": -434.47528076171875,
+      "loss": 0.4946,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.2585992813110352,
+      "rewards/margins": 0.8942249417304993,
+      "rewards/rejected": -2.1528244018554688,
       "step": 440
     },
     {
       "epoch": 0.9419152276295133,
-      "grad_norm": 42.735602427543185,
+      "grad_norm": 30.64305945908595,
       "learning_rate": 4.8708793644441086e-09,
-      "logits/chosen": -2.41510272026062,
-      "logits/rejected": -2.391575574874878,
-      "logps/chosen": -294.7413635253906,
-      "logps/rejected": -298.8751220703125,
-      "loss": 0.4984,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.45333218574523926,
-      "rewards/margins": 0.9596071243286133,
-      "rewards/rejected": -1.4129393100738525,
+      "logits/chosen": -1.6217530965805054,
+      "logits/rejected": -1.5668941736221313,
+      "logps/chosen": -399.07159423828125,
+      "logps/rejected": -491.37255859375,
+      "loss": 0.4965,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.3542201519012451,
+      "rewards/margins": 1.022148847579956,
+      "rewards/rejected": -2.376368999481201,
       "step": 450
     },
     {
       "epoch": 0.9628466771323915,
-      "grad_norm": 54.40801480075315,
+      "grad_norm": 33.639657576864714,
       "learning_rate": 1.9347820230782295e-09,
-      "logits/chosen": -2.530097484588623,
-      "logits/rejected": -2.441829204559326,
-      "logps/chosen": -284.36077880859375,
-      "logps/rejected": -262.52923583984375,
-      "loss": 0.5071,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -0.3923359513282776,
-      "rewards/margins": 0.9908507466316223,
-      "rewards/rejected": -1.3831866979599,
+      "logits/chosen": -1.7693073749542236,
+      "logits/rejected": -1.632108449935913,
+      "logps/chosen": -403.25164794921875,
+      "logps/rejected": -450.56719970703125,
+      "loss": 0.5037,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.3865845203399658,
+      "rewards/margins": 0.8871077299118042,
+      "rewards/rejected": -2.2736923694610596,
       "step": 460
     },
     {
       "epoch": 0.9837781266352695,
-      "grad_norm": 39.7809456404364,
+      "grad_norm": 24.361865004265656,
       "learning_rate": 3.2839470889836627e-10,
-      "logits/chosen": -2.536561965942383,
-      "logits/rejected": -2.4826693534851074,
-      "logps/chosen": -328.4256591796875,
-      "logps/rejected": -307.42376708984375,
-      "loss": 0.4907,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -0.22636675834655762,
-      "rewards/margins": 0.9801048040390015,
-      "rewards/rejected": -1.2064714431762695,
+      "logits/chosen": -1.7675716876983643,
+      "logits/rejected": -1.6869328022003174,
+      "logps/chosen": -416.3556213378906,
+      "logps/rejected": -471.928466796875,
+      "loss": 0.4873,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.273482084274292,
+      "rewards/margins": 0.8372520208358765,
+      "rewards/rejected": -2.110734224319458,
       "step": 470
     },
     {
       "epoch": 0.9984301412872841,
-      "eval_logits/chosen": -2.519351005554199,
-      "eval_logits/rejected": -2.4782261848449707,
-      "eval_logps/chosen": -271.4703674316406,
-      "eval_logps/rejected": -289.6701354980469,
-      "eval_loss": 0.5136305093765259,
-      "eval_rewards/accuracies": 0.75390625,
-      "eval_rewards/chosen": -0.3157782554626465,
-      "eval_rewards/margins": 0.9721657633781433,
-      "eval_rewards/rejected": -1.2879440784454346,
-      "eval_runtime": 168.3617,
-      "eval_samples_per_second": 11.879,
-      "eval_steps_per_second": 0.19,
+      "eval_logits/chosen": -1.7495195865631104,
+      "eval_logits/rejected": -1.6837695837020874,
+      "eval_logps/chosen": -377.28125,
+      "eval_logps/rejected": -477.6671142578125,
+      "eval_loss": 0.5006999969482422,
+      "eval_rewards/accuracies": 0.78515625,
+      "eval_rewards/chosen": -1.229191780090332,
+      "eval_rewards/margins": 0.990763247013092,
+      "eval_rewards/rejected": -2.2199549674987793,
+      "eval_runtime": 169.175,
+      "eval_samples_per_second": 11.822,
+      "eval_steps_per_second": 0.189,
       "step": 477
     },
     {
       "epoch": 0.9984301412872841,
       "step": 477,
       "total_flos": 0.0,
-      "train_loss": 0.5435615640516301,
-      "train_runtime": 13423.4265,
-      "train_samples_per_second": 4.554,
-      "train_steps_per_second": 0.036
+      "train_loss": 0.5402372428206278,
+      "train_runtime": 15378.7164,
+      "train_samples_per_second": 3.975,
+      "train_steps_per_second": 0.031
     }
   ],
   "logging_steps": 10,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:23a1a5c0da9cac08ae293366ab6c2e2a3363e1e05ef48b7518f77856b4c57cae
+oid sha256:11602986cd91a6bde62fe71e5cd1bcff4f3db8eb31d311ac460622a82d0e5718
 size 7672