Training in progress, epoch 3, checkpoint

last-checkpoint/model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3da40a5e2cd69e35db1c4bfd34661796a25ac71a510cbccbfb7b36b7caf9025a
+oid sha256:5bc528830c6100da3c0dd9010b862efe600ee6d7d54f51d60902108db6c72960
 size 4903351912

last-checkpoint/model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d3d4d922ecb0fbc025a2f903f492c8baa33ea8cceb3dc068450e7e0b6518684e
+oid sha256:40a843b0966455dd6d30dccf61fb993e15baeaeeaa7700f2b4813e16cccfbe4b
 size 4947570872

last-checkpoint/model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2c5d33e285bcbf98218ba55bce7efbdd306b8046089737b99aa4fe5c5e7ebea5
+oid sha256:8d30cb4c06c8541c562268facba597713984c9595ed4c681a6756bd388ec6352
 size 4962221464

last-checkpoint/model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:261c82a71bbdefce5c6b82990717f18e2f9cb6c02b8e8a09188d4f81d1e757ea
+oid sha256:4fc934496e8de7190d32b53a19a5f6c70a705b0f9fb5201734e3ec4eb86b9d93
 size 3670322200

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ca3ba1546a764910d034f0ce3e93370a84a16db802bc17eb73ebc48babbb424c
+oid sha256:e5f736309afd4efe1b8dcd7e05ccf5391826fff7c08d993abb1abf26e65aa1a6
 size 36967230034

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:74386f26f36ed67f56395205881e5db2d0c28ffcbeed50dd95b28771d2dac588
+oid sha256:06fea830cf5ad73ec00d500ea6fb952740ac936f18e93fa2d32abde1ea3ead92
 size 15984

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:41c88f9de084200454883a13c3717941ea3fd433e2f8735507fc30611f9c5501
+oid sha256:be561d1df19be227394d8ea607c54262a06c9bf880af0aa5e04a52596a2a6cb0
 size 15984

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:965b00d4cb4710ebab57c8787b9925bb3f77b8eeba94a186ec4bc1c2f326ef3f
+oid sha256:03f3e24417a59435f5a8450a4aeb0f09cc92734b5c3b45a0701b2c043c415c05
 size 15984

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d5dc374b8b9a4c45c950f9d136feab85a767081fa59f0c7d68ed3a62060c4949
+oid sha256:2bea02744c29f30024590ab1629a0e7b7dabbf1e8476456c2e7c5ce46dc35c28
 size 15984

last-checkpoint/rng_state_4.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5c7c212fb779217f1edac0baf44f67b608eefc1e0e4e3f5a9dd7eb557032c1bc
+oid sha256:041be966454b60c86af576fc1eb7f34189114689abff8f9622b947110f7334c8
 size 15984

last-checkpoint/rng_state_5.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:86e1effd626ce1e95dd68a0c8089fe19218f2b24dfe9e45ed2cab1c0ebc10ba1
+oid sha256:b85766f6596d15a810177d77dd259d9b50588cf100ec5f8ebff5fed881d57957
 size 15984

last-checkpoint/rng_state_6.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:799cc83f60dfc1c4243cfd6403592112414a2eec494e6832f10221c96ff62c20
+oid sha256:8be75d04b1ebe614241b88fd010a5dda1b7bf703c00c6ebe310ca07975830fe7
 size 15984

last-checkpoint/rng_state_7.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:586777c398770c3255d3a1f48c7fef44ea9d89117c627c9ea490e16bfd9a49ba
+oid sha256:4699833a7ab4cb692996ef7567f934c0bac79d6a067963a873f89a38e412bd48
 size 15984

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5713040e0c397c4c3f4a5dd5b82184ec7e9ce43fb4fc625e2c829d6f4609bb2d
+oid sha256:2a4821369fa96c0ec5ed7b70094f8dd7c4588e39d171b9173542d45daf7d2dc0
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 2.0,
+  "epoch": 3.0,
   "eval_steps": 300,
-  "global_step": 546,
+  "global_step": 819,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -785,6 +785,392 @@
       "learning_rate": 5.044412534486873e-07,
       "loss": 4.7148,
       "step": 545
+    },
+    {
+      "epoch": 2.0146520146520146,
+      "grad_norm": 3.71875,
+      "learning_rate": 4.878545062680026e-07,
+      "loss": 4.6767,
+      "step": 550
+    },
+    {
+      "epoch": 2.032967032967033,
+      "grad_norm": 3.453125,
+      "learning_rate": 4.7145660742090575e-07,
+      "loss": 4.706,
+      "step": 555
+    },
+    {
+      "epoch": 2.051282051282051,
+      "grad_norm": 4.0,
+      "learning_rate": 4.5525360346234907e-07,
+      "loss": 4.7142,
+      "step": 560
+    },
+    {
+      "epoch": 2.06959706959707,
+      "grad_norm": 3.546875,
+      "learning_rate": 4.392514690818193e-07,
+      "loss": 4.7053,
+      "step": 565
+    },
+    {
+      "epoch": 2.087912087912088,
+      "grad_norm": 3.765625,
+      "learning_rate": 4.2345610490022996e-07,
+      "loss": 4.6978,
+      "step": 570
+    },
+    {
+      "epoch": 2.1062271062271063,
+      "grad_norm": 3.53125,
+      "learning_rate": 4.078733352941321e-07,
+      "loss": 4.7147,
+      "step": 575
+    },
+    {
+      "epoch": 2.1245421245421245,
+      "grad_norm": 3.546875,
+      "learning_rate": 3.925089062480339e-07,
+      "loss": 4.6861,
+      "step": 580
+    },
+    {
+      "epoch": 2.142857142857143,
+      "grad_norm": 3.703125,
+      "learning_rate": 3.77368483235628e-07,
+      "loss": 4.725,
+      "step": 585
+    },
+    {
+      "epoch": 2.161172161172161,
+      "grad_norm": 3.453125,
+      "learning_rate": 3.6245764913070875e-07,
+      "loss": 4.7223,
+      "step": 590
+    },
+    {
+      "epoch": 2.1794871794871793,
+      "grad_norm": 3.578125,
+      "learning_rate": 3.477819021485447e-07,
+      "loss": 4.7254,
+      "step": 595
+    },
+    {
+      "epoch": 2.197802197802198,
+      "grad_norm": 3.578125,
+      "learning_rate": 3.333466538184674e-07,
+      "loss": 4.7074,
+      "step": 600
+    },
+    {
+      "epoch": 2.197802197802198,
+      "eval_loss": 4.718916893005371,
+      "eval_runtime": 14.5284,
+      "eval_samples_per_second": 36.412,
+      "eval_steps_per_second": 1.17,
+      "step": 600
+    },
+    {
+      "epoch": 2.2161172161172162,
+      "grad_norm": 3.546875,
+      "learning_rate": 3.1915722698842874e-07,
+      "loss": 4.705,
+      "step": 605
+    },
+    {
+      "epoch": 2.2344322344322345,
+      "grad_norm": 3.734375,
+      "learning_rate": 3.0521885386225344e-07,
+      "loss": 4.7118,
+      "step": 610
+    },
+    {
+      "epoch": 2.2527472527472527,
+      "grad_norm": 3.703125,
+      "learning_rate": 2.9153667407032066e-07,
+      "loss": 4.7285,
+      "step": 615
+    },
+    {
+      "epoch": 2.271062271062271,
+      "grad_norm": 3.59375,
+      "learning_rate": 2.7811573277437603e-07,
+      "loss": 4.681,
+      "step": 620
+    },
+    {
+      "epoch": 2.2893772893772892,
+      "grad_norm": 3.546875,
+      "learning_rate": 2.649609788071836e-07,
+      "loss": 4.707,
+      "step": 625
+    },
+    {
+      "epoch": 2.3076923076923075,
+      "grad_norm": 3.46875,
+      "learning_rate": 2.520772628476919e-07,
+      "loss": 4.7278,
+      "step": 630
+    },
+    {
+      "epoch": 2.326007326007326,
+      "grad_norm": 3.59375,
+      "learning_rate": 2.394693356323997e-07,
+      "loss": 4.7202,
+      "step": 635
+    },
+    {
+      "epoch": 2.3443223443223444,
+      "grad_norm": 3.609375,
+      "learning_rate": 2.2714184620356826e-07,
+      "loss": 4.706,
+      "step": 640
+    },
+    {
+      "epoch": 2.3626373626373627,
+      "grad_norm": 3.5,
+      "learning_rate": 2.150993401949376e-07,
+      "loss": 4.732,
+      "step": 645
+    },
+    {
+      "epoch": 2.380952380952381,
+      "grad_norm": 4.46875,
+      "learning_rate": 2.0334625815557026e-07,
+      "loss": 4.6981,
+      "step": 650
+    },
+    {
+      "epoch": 2.399267399267399,
+      "grad_norm": 3.5625,
+      "learning_rate": 1.9188693391244438e-07,
+      "loss": 4.7334,
+      "step": 655
+    },
+    {
+      "epoch": 2.4175824175824174,
+      "grad_norm": 3.609375,
+      "learning_rate": 1.8072559297240097e-07,
+      "loss": 4.7284,
+      "step": 660
+    },
+    {
+      "epoch": 2.435897435897436,
+      "grad_norm": 3.515625,
+      "learning_rate": 1.6986635096403212e-07,
+      "loss": 4.7189,
+      "step": 665
+    },
+    {
+      "epoch": 2.4542124542124544,
+      "grad_norm": 3.484375,
+      "learning_rate": 1.5931321212008465e-07,
+      "loss": 4.7007,
+      "step": 670
+    },
+    {
+      "epoch": 2.4725274725274726,
+      "grad_norm": 3.53125,
+      "learning_rate": 1.490700678009421e-07,
+      "loss": 4.6923,
+      "step": 675
+    },
+    {
+      "epoch": 2.490842490842491,
+      "grad_norm": 3.546875,
+      "learning_rate": 1.3914069505972482e-07,
+      "loss": 4.7265,
+      "step": 680
+    },
+    {
+      "epoch": 2.509157509157509,
+      "grad_norm": 3.6875,
+      "learning_rate": 1.2952875524954232e-07,
+      "loss": 4.7318,
+      "step": 685
+    },
+    {
+      "epoch": 2.5274725274725274,
+      "grad_norm": 3.546875,
+      "learning_rate": 1.2023779267340563e-07,
+      "loss": 4.7144,
+      "step": 690
+    },
+    {
+      "epoch": 2.5457875457875456,
+      "grad_norm": 3.609375,
+      "learning_rate": 1.112712332773038e-07,
+      "loss": 4.6979,
+      "step": 695
+    },
+    {
+      "epoch": 2.564102564102564,
+      "grad_norm": 3.5625,
+      "learning_rate": 1.026323833869206e-07,
+      "loss": 4.6933,
+      "step": 700
+    },
+    {
+      "epoch": 2.5824175824175826,
+      "grad_norm": 3.59375,
+      "learning_rate": 9.432442848846289e-08,
+      "loss": 4.731,
+      "step": 705
+    },
+    {
+      "epoch": 2.600732600732601,
+      "grad_norm": 3.53125,
+      "learning_rate": 8.63504320540438e-08,
+      "loss": 4.7246,
+      "step": 710
+    },
+    {
+      "epoch": 2.619047619047619,
+      "grad_norm": 3.53125,
+      "learning_rate": 7.871333441206052e-08,
+      "loss": 4.7311,
+      "step": 715
+    },
+    {
+      "epoch": 2.6373626373626373,
+      "grad_norm": 3.53125,
+      "learning_rate": 7.141595166297832e-08,
+      "loss": 4.7213,
+      "step": 720
+    },
+    {
+      "epoch": 2.6556776556776556,
+      "grad_norm": 3.484375,
+      "learning_rate": 6.446097464092248e-08,
+      "loss": 4.6983,
+      "step": 725
+    },
+    {
+      "epoch": 2.6739926739926743,
+      "grad_norm": 3.546875,
+      "learning_rate": 5.78509679214616e-08,
+      "loss": 4.6839,
+      "step": 730
+    },
+    {
+      "epoch": 2.6923076923076925,
+      "grad_norm": 3.5625,
+      "learning_rate": 5.1588368875946864e-08,
+      "loss": 4.7098,
+      "step": 735
+    },
+    {
+      "epoch": 2.7106227106227108,
+      "grad_norm": 3.65625,
+      "learning_rate": 4.567548677275601e-08,
+      "loss": 4.6954,
+      "step": 740
+    },
+    {
+      "epoch": 2.728937728937729,
+      "grad_norm": 3.578125,
+      "learning_rate": 4.0114501925775925e-08,
+      "loss": 4.7284,
+      "step": 745
+    },
+    {
+      "epoch": 2.7472527472527473,
+      "grad_norm": 3.765625,
+      "learning_rate": 3.490746489043317e-08,
+      "loss": 4.7099,
+      "step": 750
+    },
+    {
+      "epoch": 2.7655677655677655,
+      "grad_norm": 3.5625,
+      "learning_rate": 3.005629570757373e-08,
+      "loss": 4.7144,
+      "step": 755
+    },
+    {
+      "epoch": 2.7838827838827838,
+      "grad_norm": 3.625,
+      "learning_rate": 2.5562783195467675e-08,
+      "loss": 4.6949,
+      "step": 760
+    },
+    {
+      "epoch": 2.802197802197802,
+      "grad_norm": 3.515625,
+      "learning_rate": 2.1428584290201114e-08,
+      "loss": 4.7045,
+      "step": 765
+    },
+    {
+      "epoch": 2.8205128205128203,
+      "grad_norm": 3.703125,
+      "learning_rate": 1.7655223434698053e-08,
+      "loss": 4.688,
+      "step": 770
+    },
+    {
+      "epoch": 2.838827838827839,
+      "grad_norm": 3.578125,
+      "learning_rate": 1.4244092016597931e-08,
+      "loss": 4.7177,
+      "step": 775
+    },
+    {
+      "epoch": 2.857142857142857,
+      "grad_norm": 3.59375,
+      "learning_rate": 1.11964478551958e-08,
+      "loss": 4.7073,
+      "step": 780
+    },
+    {
+      "epoch": 2.8754578754578755,
+      "grad_norm": 3.453125,
+      "learning_rate": 8.513414737635005e-09,
+      "loss": 4.7176,
+      "step": 785
+    },
+    {
+      "epoch": 2.8937728937728937,
+      "grad_norm": 3.515625,
+      "learning_rate": 6.1959820045215385e-09,
+      "loss": 4.6779,
+      "step": 790
+    },
+    {
+      "epoch": 2.912087912087912,
+      "grad_norm": 3.484375,
+      "learning_rate": 4.245004185115752e-09,
+      "loss": 4.7052,
+      "step": 795
+    },
+    {
+      "epoch": 2.9304029304029307,
+      "grad_norm": 3.484375,
+      "learning_rate": 2.661200682232745e-09,
+      "loss": 4.7179,
+      "step": 800
+    },
+    {
+      "epoch": 2.948717948717949,
+      "grad_norm": 3.640625,
+      "learning_rate": 1.4451555069708853e-09,
+      "loss": 4.6552,
+      "step": 805
+    },
+    {
+      "epoch": 2.967032967032967,
+      "grad_norm": 4.21875,
+      "learning_rate": 5.973170633631897e-10,
+      "loss": 4.6979,
+      "step": 810
+    },
+    {
+      "epoch": 2.9853479853479854,
+      "grad_norm": 3.578125,
+      "learning_rate": 1.1799798303335772e-10,
+      "loss": 4.7113,
+      "step": 815
     }
   ],
   "logging_steps": 5,
@@ -799,12 +1185,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 3.574344656441311e+18,
+  "total_flos": 5.361516984661967e+18,
   "train_batch_size": 8,
   "trial_name": null,
   "trial_params": null