Training in progress, step 264, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:92aa718963ff1a158ab63158709261c7329af2ae34ebe9805357cdb7a33e38de
+oid sha256:b6b7fbaf2d6a6e1654728bf2b64ff7a097f615d5247c146dd31d3eccfa8fc30f
 size 763470136

last-checkpoint/global_step264/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ca144ba18752225282c0f4544d978254c2ae4f35d68cae745a609c918846e1f
+size 289065424

last-checkpoint/global_step264/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:614e9ee88e9bcb3a45429c59b2bf2769ba433567f51363c7a4253b475b21e6bc
+size 289065424

last-checkpoint/global_step264/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d655ebb2146d9487a8d09cd971ceb207ee79d6d7c42ab932971bb75ba93e940c
+size 289065424

last-checkpoint/global_step264/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89eae13e8c0ba60e40a9c3d3f11b76646b83ba197f6abf0aba73b689661edd6c
+size 289065424

last-checkpoint/global_step264/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0557f6a14c69ef21c2ed3a5c48be9f7a3a47f102c4b16e6db9df1e8036a73a5
+size 289065424

last-checkpoint/global_step264/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d0052e580fe2d1f70992b2fcc233627549397786188980f2ec6a856d84d45f9
+size 289065424

last-checkpoint/global_step264/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67db40a4c668372a5d51fa8aad0bf99b67dda6f1acdcfc128369378fd063a309
+size 289065424

last-checkpoint/global_step264/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca273d1927a1a34aa37630afe3fae7e5b729674eeb60cf491b164d2415105d78
+size 289065424

last-checkpoint/global_step264/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef56386f8cf47768c633eb8b8ab10076925c3b5e457f1548072d50346f8b468d
+size 348711830

last-checkpoint/global_step264/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5bfd7e5ba0884a15f626b0c26345601fddc4900a760abdb231c701069e165d2
+size 348711830

last-checkpoint/global_step264/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67d112660240266bc61d3417f0745f1145af0c63963c37f516b89c36b4985ece
+size 348711830

last-checkpoint/global_step264/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36d966bcd023af6eda0fb4a38472fa53c83aa9a27937caf21dd846c3fd6f9274
+size 348711830

last-checkpoint/global_step264/zero_pp_rank_4_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c28b986c53d2c4dc45b78474b2e13f40f55e7da315b0b108f035a72e86cd9308
+size 348711830

last-checkpoint/global_step264/zero_pp_rank_5_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e59b35e6b3d5165fdaae19137b371d5a6458b844c4a65494b02eba621e681844
+size 348711830

last-checkpoint/global_step264/zero_pp_rank_6_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:881ffb5534b69c57a2e00bbfd8c40881c2f08386b7a27b1f3c4179356f366e18
+size 348711830

last-checkpoint/global_step264/zero_pp_rank_7_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63a9082d03b443436f3f7c312f3eae46a679986eef5d0a24630c726e3b8afa34
+size 348711830

last-checkpoint/latest

@@ -1 +1 @@
-global_step198
+global_step264

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b659790db5a549bc94a52bd0661c6c5e6c19beea5b259996f6ed9fe2149516f2
+oid sha256:756188867614fe144ce7bb4100b8fdc4a53793718efdbfd597ab9a7af1127cb3
 size 15920

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4fedd424305bc76c60abafd8b0806d3107fa0fd9dcab69abdd8a175961c5d292
+oid sha256:9326dda8ccb88256fea16bdb08bf3d8ee2d7890d74941621ea0ae79baad53127
 size 15920

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:018b23a19c9fcba72d4cdfa2c9fc3962ed3bc3bd0e06e1ebeb979a60bbcca587
+oid sha256:dc7ea8107c02800ceda5d3219d8139cc0c46423c770369f8d482750d2ee66b59
 size 15920

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:655e1d8eba47928d19c120d020c1358f82da6b7b643dec3c9fd55e5052edd4fe
+oid sha256:5100775819feb4598b355aaf5ae7a2d05f1e6c33d82585848692501430716b79
 size 15920

last-checkpoint/rng_state_4.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0051f5950d3497fd49cd25af996fe01c32a4128ba6dc3623a168e00768ef4bd5
+oid sha256:375d7beb01cab64b2715fb3d805593967127e2433072776577d1a22535bc71f6
 size 15920

last-checkpoint/rng_state_5.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:81af3423c5f1a9239eebc7b36cb6e6db3f9862f7b90cd7560fef2590ee1d68d0
+oid sha256:be4bc162636adeba1331e40da73f3fb1fde2fb44472545ff46bc3e2a6588d115
 size 15920

last-checkpoint/rng_state_6.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:493f5e097b662c4de6f929779988d574e0855983f464da2bbac2cf6d59691a7a
+oid sha256:9e4ec9613f9c318e718457c34ba482fb1b487745cd80d6e26c4479f47030f964
 size 15920

last-checkpoint/rng_state_7.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:47313163e11ebbda29b8bf91fd61cb4b29fc84b8ec482325f230809ff25c6426
+oid sha256:e6c5785e3656da35a0034b82ee38c2b260ac87d57dc93498957445739f27c017
 size 15920

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ced86f3a1c08cecda79ca695145fe007ebcfd4f2f8962847f6a9d9d58b4b557b
+oid sha256:26eca587873b25805521ebb406b132a4ba3e54d5f099d35d9e497769da91dcd6
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.6073619631901841,
+  "epoch": 0.8098159509202454,
   "eval_steps": 66,
-  "global_step": 198,
+  "global_step": 264,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -1425,6 +1425,476 @@
       "eval_samples_per_second": 1.793,
       "eval_steps_per_second": 0.126,
       "step": 198
+    },
+    {
+      "epoch": 0.6104294478527608,
+      "grad_norm": 0.31755022515076264,
+      "learning_rate": 4.313016905898286e-05,
+      "loss": 1.8861,
+      "step": 199
+    },
+    {
+      "epoch": 0.6134969325153374,
+      "grad_norm": 0.37514333178831394,
+      "learning_rate": 4.268516544675628e-05,
+      "loss": 1.9366,
+      "step": 200
+    },
+    {
+      "epoch": 0.6165644171779141,
+      "grad_norm": 0.2768732078613857,
+      "learning_rate": 4.224145985535202e-05,
+      "loss": 1.8781,
+      "step": 201
+    },
+    {
+      "epoch": 0.6196319018404908,
+      "grad_norm": 0.385983235346578,
+      "learning_rate": 4.1799099052681934e-05,
+      "loss": 1.9089,
+      "step": 202
+    },
+    {
+      "epoch": 0.6226993865030674,
+      "grad_norm": 0.34929147929166254,
+      "learning_rate": 4.135812966491305e-05,
+      "loss": 1.9409,
+      "step": 203
+    },
+    {
+      "epoch": 0.6257668711656442,
+      "grad_norm": 0.3448745967701562,
+      "learning_rate": 4.091859817155307e-05,
+      "loss": 1.8935,
+      "step": 204
+    },
+    {
+      "epoch": 0.6288343558282209,
+      "grad_norm": 0.24777573443198542,
+      "learning_rate": 4.048055090055125e-05,
+      "loss": 1.9007,
+      "step": 205
+    },
+    {
+      "epoch": 0.6319018404907976,
+      "grad_norm": 0.33163324355956286,
+      "learning_rate": 4.004403402341532e-05,
+      "loss": 1.8816,
+      "step": 206
+    },
+    {
+      "epoch": 0.6349693251533742,
+      "grad_norm": 0.6161345209342699,
+      "learning_rate": 3.960909355034491e-05,
+      "loss": 1.8952,
+      "step": 207
+    },
+    {
+      "epoch": 0.6380368098159509,
+      "grad_norm": 0.29863513222265725,
+      "learning_rate": 3.917577532538185e-05,
+      "loss": 1.8622,
+      "step": 208
+    },
+    {
+      "epoch": 0.6411042944785276,
+      "grad_norm": 0.23544641297651625,
+      "learning_rate": 3.8744125021578126e-05,
+      "loss": 1.9098,
+      "step": 209
+    },
+    {
+      "epoch": 0.6441717791411042,
+      "grad_norm": 0.29701664972205183,
+      "learning_rate": 3.831418813618177e-05,
+      "loss": 1.8963,
+      "step": 210
+    },
+    {
+      "epoch": 0.647239263803681,
+      "grad_norm": 0.2608462550147094,
+      "learning_rate": 3.788600998584135e-05,
+      "loss": 1.9425,
+      "step": 211
+    },
+    {
+      "epoch": 0.6503067484662577,
+      "grad_norm": 0.2753794235571961,
+      "learning_rate": 3.7459635701829435e-05,
+      "loss": 1.9312,
+      "step": 212
+    },
+    {
+      "epoch": 0.6533742331288344,
+      "grad_norm": 0.40974803557689143,
+      "learning_rate": 3.703511022528562e-05,
+      "loss": 1.8992,
+      "step": 213
+    },
+    {
+      "epoch": 0.656441717791411,
+      "grad_norm": 0.24030236007607908,
+      "learning_rate": 3.6612478302479594e-05,
+      "loss": 1.9326,
+      "step": 214
+    },
+    {
+      "epoch": 0.6595092024539877,
+      "grad_norm": 0.4383608820045659,
+      "learning_rate": 3.619178448009477e-05,
+      "loss": 1.932,
+      "step": 215
+    },
+    {
+      "epoch": 0.6625766871165644,
+      "grad_norm": 0.5102560092350799,
+      "learning_rate": 3.5773073100532874e-05,
+      "loss": 1.8956,
+      "step": 216
+    },
+    {
+      "epoch": 0.6656441717791411,
+      "grad_norm": 0.36274812580727284,
+      "learning_rate": 3.535638829724019e-05,
+      "loss": 1.8919,
+      "step": 217
+    },
+    {
+      "epoch": 0.6687116564417178,
+      "grad_norm": 0.23488730500365318,
+      "learning_rate": 3.494177399005578e-05,
+      "loss": 1.9158,
+      "step": 218
+    },
+    {
+      "epoch": 0.6717791411042945,
+      "grad_norm": 0.3741689726256645,
+      "learning_rate": 3.452927388058206e-05,
+      "loss": 1.9423,
+      "step": 219
+    },
+    {
+      "epoch": 0.6748466257668712,
+      "grad_norm": 0.25651456348082824,
+      "learning_rate": 3.411893144757866e-05,
+      "loss": 1.8415,
+      "step": 220
+    },
+    {
+      "epoch": 0.6779141104294478,
+      "grad_norm": 0.2612828905023135,
+      "learning_rate": 3.3710789942379556e-05,
+      "loss": 1.9472,
+      "step": 221
+    },
+    {
+      "epoch": 0.6809815950920245,
+      "grad_norm": 0.25469935789428655,
+      "learning_rate": 3.33048923843343e-05,
+      "loss": 1.949,
+      "step": 222
+    },
+    {
+      "epoch": 0.6840490797546013,
+      "grad_norm": 0.23410106434735667,
+      "learning_rate": 3.2901281556273646e-05,
+      "loss": 1.8963,
+      "step": 223
+    },
+    {
+      "epoch": 0.6871165644171779,
+      "grad_norm": 0.25811790889112224,
+      "learning_rate": 3.250000000000001e-05,
+      "loss": 1.8488,
+      "step": 224
+    },
+    {
+      "epoch": 0.6901840490797546,
+      "grad_norm": 0.2701258126507899,
+      "learning_rate": 3.210109001180358e-05,
+      "loss": 1.9429,
+      "step": 225
+    },
+    {
+      "epoch": 0.6932515337423313,
+      "grad_norm": 0.27336206551312103,
+      "learning_rate": 3.170459363800409e-05,
+      "loss": 1.9063,
+      "step": 226
+    },
+    {
+      "epoch": 0.696319018404908,
+      "grad_norm": 0.30139160569284024,
+      "learning_rate": 3.1310552670518986e-05,
+      "loss": 1.9182,
+      "step": 227
+    },
+    {
+      "epoch": 0.6993865030674846,
+      "grad_norm": 0.23370917590561624,
+      "learning_rate": 3.0919008642458494e-05,
+      "loss": 1.9541,
+      "step": 228
+    },
+    {
+      "epoch": 0.7024539877300614,
+      "grad_norm": 0.22222235132591592,
+      "learning_rate": 3.053000282374781e-05,
+      "loss": 1.8864,
+      "step": 229
+    },
+    {
+      "epoch": 0.7055214723926381,
+      "grad_norm": 0.27873390973935386,
+      "learning_rate": 3.014357621677724e-05,
+      "loss": 1.8852,
+      "step": 230
+    },
+    {
+      "epoch": 0.7085889570552147,
+      "grad_norm": 0.3108583744507131,
+      "learning_rate": 2.9759769552080376e-05,
+      "loss": 1.8663,
+      "step": 231
+    },
+    {
+      "epoch": 0.7116564417177914,
+      "grad_norm": 0.30913975922284836,
+      "learning_rate": 2.93786232840409e-05,
+      "loss": 1.9404,
+      "step": 232
+    },
+    {
+      "epoch": 0.7147239263803681,
+      "grad_norm": 0.28385532690084997,
+      "learning_rate": 2.90001775866287e-05,
+      "loss": 1.9023,
+      "step": 233
+    },
+    {
+      "epoch": 0.7177914110429447,
+      "grad_norm": 0.2808200803737186,
+      "learning_rate": 2.8624472349165355e-05,
+      "loss": 1.9192,
+      "step": 234
+    },
+    {
+      "epoch": 0.7208588957055214,
+      "grad_norm": 0.23648694756886077,
+      "learning_rate": 2.8251547172119603e-05,
+      "loss": 2.0132,
+      "step": 235
+    },
+    {
+      "epoch": 0.7239263803680982,
+      "grad_norm": 0.6069490067148141,
+      "learning_rate": 2.7881441362933468e-05,
+      "loss": 1.8395,
+      "step": 236
+    },
+    {
+      "epoch": 0.7269938650306749,
+      "grad_norm": 0.3350257794257116,
+      "learning_rate": 2.751419393187905e-05,
+      "loss": 1.8667,
+      "step": 237
+    },
+    {
+      "epoch": 0.7300613496932515,
+      "grad_norm": 0.232164276820369,
+      "learning_rate": 2.7149843587946744e-05,
+      "loss": 1.8656,
+      "step": 238
+    },
+    {
+      "epoch": 0.7331288343558282,
+      "grad_norm": 0.38356734047420593,
+      "learning_rate": 2.6788428734765224e-05,
+      "loss": 1.9048,
+      "step": 239
+    },
+    {
+      "epoch": 0.7361963190184049,
+      "grad_norm": 0.2618731826165273,
+      "learning_rate": 2.642998746655348e-05,
+      "loss": 1.9783,
+      "step": 240
+    },
+    {
+      "epoch": 0.7392638036809815,
+      "grad_norm": 0.6648822511934657,
+      "learning_rate": 2.6074557564105727e-05,
+      "loss": 1.9043,
+      "step": 241
+    },
+    {
+      "epoch": 0.7423312883435583,
+      "grad_norm": 0.27175163581016115,
+      "learning_rate": 2.5722176490809118e-05,
+      "loss": 1.9585,
+      "step": 242
+    },
+    {
+      "epoch": 0.745398773006135,
+      "grad_norm": 0.3925966681047075,
+      "learning_rate": 2.5372881388694912e-05,
+      "loss": 1.8515,
+      "step": 243
+    },
+    {
+      "epoch": 0.7484662576687117,
+      "grad_norm": 0.37190935188206453,
+      "learning_rate": 2.5026709074523748e-05,
+      "loss": 1.9688,
+      "step": 244
+    },
+    {
+      "epoch": 0.7515337423312883,
+      "grad_norm": 0.2257138379202953,
+      "learning_rate": 2.4683696035904928e-05,
+      "loss": 1.9486,
+      "step": 245
+    },
+    {
+      "epoch": 0.754601226993865,
+      "grad_norm": 0.2274145468605237,
+      "learning_rate": 2.434387842745056e-05,
+      "loss": 1.9302,
+      "step": 246
+    },
+    {
+      "epoch": 0.7576687116564417,
+      "grad_norm": 0.5126959359452324,
+      "learning_rate": 2.400729206696477e-05,
+      "loss": 1.9443,
+      "step": 247
+    },
+    {
+      "epoch": 0.7607361963190185,
+      "grad_norm": 0.2551304692334095,
+      "learning_rate": 2.3673972431668306e-05,
+      "loss": 2.009,
+      "step": 248
+    },
+    {
+      "epoch": 0.7638036809815951,
+      "grad_norm": 0.4447523876477682,
+      "learning_rate": 2.334395465445926e-05,
+      "loss": 1.8468,
+      "step": 249
+    },
+    {
+      "epoch": 0.7668711656441718,
+      "grad_norm": 0.2657558360669318,
+      "learning_rate": 2.3017273520209882e-05,
+      "loss": 1.8886,
+      "step": 250
+    },
+    {
+      "epoch": 0.7699386503067485,
+      "grad_norm": 0.37573420755761094,
+      "learning_rate": 2.2693963462100117e-05,
+      "loss": 1.8663,
+      "step": 251
+    },
+    {
+      "epoch": 0.7730061349693251,
+      "grad_norm": 0.26075506564879214,
+      "learning_rate": 2.2374058557988336e-05,
+      "loss": 1.909,
+      "step": 252
+    },
+    {
+      "epoch": 0.7760736196319018,
+      "grad_norm": 0.2951446457265513,
+      "learning_rate": 2.2057592526819353e-05,
+      "loss": 1.9362,
+      "step": 253
+    },
+    {
+      "epoch": 0.7791411042944786,
+      "grad_norm": 0.24420003456766767,
+      "learning_rate": 2.1744598725070347e-05,
+      "loss": 1.9134,
+      "step": 254
+    },
+    {
+      "epoch": 0.7822085889570553,
+      "grad_norm": 0.2563261666147908,
+      "learning_rate": 2.143511014323506e-05,
+      "loss": 1.9569,
+      "step": 255
+    },
+    {
+      "epoch": 0.7852760736196319,
+      "grad_norm": 0.27427716272900493,
+      "learning_rate": 2.11291594023464e-05,
+      "loss": 1.8982,
+      "step": 256
+    },
+    {
+      "epoch": 0.7883435582822086,
+      "grad_norm": 0.4685271777395839,
+      "learning_rate": 2.082677875053818e-05,
+      "loss": 1.9256,
+      "step": 257
+    },
+    {
+      "epoch": 0.7914110429447853,
+      "grad_norm": 0.3080424306042412,
+      "learning_rate": 2.0528000059645997e-05,
+      "loss": 1.9154,
+      "step": 258
+    },
+    {
+      "epoch": 0.7944785276073619,
+      "grad_norm": 0.2672783439075976,
+      "learning_rate": 2.023285482184785e-05,
+      "loss": 1.9574,
+      "step": 259
+    },
+    {
+      "epoch": 0.7975460122699386,
+      "grad_norm": 0.3311914465278651,
+      "learning_rate": 1.994137414634483e-05,
+      "loss": 1.9133,
+      "step": 260
+    },
+    {
+      "epoch": 0.8006134969325154,
+      "grad_norm": 0.7675438620825049,
+      "learning_rate": 1.9653588756082064e-05,
+      "loss": 1.892,
+      "step": 261
+    },
+    {
+      "epoch": 0.803680981595092,
+      "grad_norm": 0.2757310062776552,
+      "learning_rate": 1.9369528984510394e-05,
+      "loss": 1.9087,
+      "step": 262
+    },
+    {
+      "epoch": 0.8067484662576687,
+      "grad_norm": 0.24797296946202665,
+      "learning_rate": 1.9089224772389225e-05,
+      "loss": 1.8836,
+      "step": 263
+    },
+    {
+      "epoch": 0.8098159509202454,
+      "grad_norm": 0.41244928985184576,
+      "learning_rate": 1.881270566463062e-05,
+      "loss": 1.9094,
+      "step": 264
+    },
+    {
+      "epoch": 0.8098159509202454,
+      "eval_loss": 2.593792200088501,
+      "eval_runtime": 55.7303,
+      "eval_samples_per_second": 1.794,
+      "eval_steps_per_second": 0.126,
+      "step": 264
     }
   ],
   "logging_steps": 1,
@@ -1444,7 +1914,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 216215096131584.0,
+  "total_flos": 288286794842112.0,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null