Upload folder using huggingface_hub

checkpoints/main.log

@@ -1091,3 +1091,60 @@ Mixed precision type: bf16
 [2024-08-11 11:57:38,118][Main][INFO] - [train] Step 50100 out of 80000 | Loss --> 1.853 | Grad_l2 --> 0.305 | Weights_l2 --> 9101.232 | Lr --> 0.003 | Seconds_per_step --> 3.388 | 
 [2024-08-11 12:00:27,137][Main][INFO] - [train] Step 50150 out of 80000 | Loss --> 1.858 | Grad_l2 --> 0.305 | Weights_l2 --> 9101.189 | Lr --> 0.003 | Seconds_per_step --> 3.380 | 
 [2024-08-11 12:03:16,714][Main][INFO] - [train] Step 50200 out of 80000 | Loss --> 1.853 | Grad_l2 --> 0.304 | Weights_l2 --> 9101.138 | Lr --> 0.003 | Seconds_per_step --> 3.392 | 
+[2024-08-11 12:06:06,587][Main][INFO] - [train] Step 50250 out of 80000 | Loss --> 1.863 | Grad_l2 --> 0.303 | Weights_l2 --> 9101.077 | Lr --> 0.003 | Seconds_per_step --> 3.397 | 
+[2024-08-11 12:08:54,840][Main][INFO] - [train] Step 50300 out of 80000 | Loss --> 1.859 | Grad_l2 --> 0.308 | Weights_l2 --> 9101.026 | Lr --> 0.003 | Seconds_per_step --> 3.365 | 
+[2024-08-11 12:11:43,859][Main][INFO] - [train] Step 50350 out of 80000 | Loss --> 1.867 | Grad_l2 --> 0.306 | Weights_l2 --> 9100.972 | Lr --> 0.003 | Seconds_per_step --> 3.380 | 
+[2024-08-11 12:14:31,964][Main][INFO] - [train] Step 50400 out of 80000 | Loss --> 1.858 | Grad_l2 --> 0.307 | Weights_l2 --> 9100.919 | Lr --> 0.003 | Seconds_per_step --> 3.362 | 
+[2024-08-11 12:17:20,154][Main][INFO] - [train] Step 50450 out of 80000 | Loss --> 1.865 | Grad_l2 --> 0.306 | Weights_l2 --> 9100.876 | Lr --> 0.003 | Seconds_per_step --> 3.364 | 
+[2024-08-11 12:20:08,016][Main][INFO] - [train] Step 50500 out of 80000 | Loss --> 1.856 | Grad_l2 --> 0.304 | Weights_l2 --> 9100.820 | Lr --> 0.003 | Seconds_per_step --> 3.357 | 
+[2024-08-11 12:22:56,654][Main][INFO] - [train] Step 50550 out of 80000 | Loss --> 1.859 | Grad_l2 --> 0.306 | Weights_l2 --> 9100.766 | Lr --> 0.003 | Seconds_per_step --> 3.373 | 
+[2024-08-11 12:25:46,183][Main][INFO] - [train] Step 50600 out of 80000 | Loss --> 1.859 | Grad_l2 --> 0.304 | Weights_l2 --> 9100.712 | Lr --> 0.003 | Seconds_per_step --> 3.391 | 
+[2024-08-11 12:28:36,862][Main][INFO] - [train] Step 50650 out of 80000 | Loss --> 1.866 | Grad_l2 --> 0.304 | Weights_l2 --> 9100.660 | Lr --> 0.003 | Seconds_per_step --> 3.414 | 
+[2024-08-11 12:31:26,475][Main][INFO] - [train] Step 50700 out of 80000 | Loss --> 1.857 | Grad_l2 --> 0.302 | Weights_l2 --> 9100.605 | Lr --> 0.003 | Seconds_per_step --> 3.392 | 
+[2024-08-11 12:34:15,202][Main][INFO] - [train] Step 50750 out of 80000 | Loss --> 1.856 | Grad_l2 --> 0.308 | Weights_l2 --> 9100.546 | Lr --> 0.003 | Seconds_per_step --> 3.375 | 
+[2024-08-11 12:37:04,603][Main][INFO] - [train] Step 50800 out of 80000 | Loss --> 1.857 | Grad_l2 --> 0.304 | Weights_l2 --> 9100.481 | Lr --> 0.003 | Seconds_per_step --> 3.388 | 
+[2024-08-11 12:39:53,849][Main][INFO] - [train] Step 50850 out of 80000 | Loss --> 1.851 | Grad_l2 --> 0.304 | Weights_l2 --> 9100.431 | Lr --> 0.003 | Seconds_per_step --> 3.385 | 
+[2024-08-11 12:42:42,323][Main][INFO] - [train] Step 50900 out of 80000 | Loss --> 1.856 | Grad_l2 --> 0.304 | Weights_l2 --> 9100.369 | Lr --> 0.003 | Seconds_per_step --> 3.369 | 
+[2024-08-11 12:45:30,712][Main][INFO] - [train] Step 50950 out of 80000 | Loss --> 1.863 | Grad_l2 --> 0.304 | Weights_l2 --> 9100.297 | Lr --> 0.003 | Seconds_per_step --> 3.368 | 
+[2024-08-11 12:48:19,531][Main][INFO] - [train] Step 51000 out of 80000 | Loss --> 1.856 | Grad_l2 --> 0.305 | Weights_l2 --> 9100.235 | Lr --> 0.003 | Seconds_per_step --> 3.376 | 
+[2024-08-11 12:51:12,887][Main][INFO] - [train] Step 51050 out of 80000 | Loss --> 1.851 | Grad_l2 --> 0.302 | Weights_l2 --> 9100.189 | Lr --> 0.003 | Seconds_per_step --> 3.467 | 
+[2024-08-11 12:54:01,929][Main][INFO] - [train] Step 51100 out of 80000 | Loss --> 1.847 | Grad_l2 --> 0.306 | Weights_l2 --> 9100.124 | Lr --> 0.003 | Seconds_per_step --> 3.381 | 
+[2024-08-11 12:56:51,344][Main][INFO] - [train] Step 51150 out of 80000 | Loss --> 1.841 | Grad_l2 --> 0.305 | Weights_l2 --> 9100.062 | Lr --> 0.003 | Seconds_per_step --> 3.388 | 
+[2024-08-11 12:59:41,233][Main][INFO] - [train] Step 51200 out of 80000 | Loss --> 1.850 | Grad_l2 --> 0.305 | Weights_l2 --> 9099.993 | Lr --> 0.003 | Seconds_per_step --> 3.398 | 
+[2024-08-11 13:02:30,398][Main][INFO] - [train] Step 51250 out of 80000 | Loss --> 1.850 | Grad_l2 --> 0.302 | Weights_l2 --> 9099.923 | Lr --> 0.003 | Seconds_per_step --> 3.383 | 
+[2024-08-11 13:05:18,484][Main][INFO] - [train] Step 51300 out of 80000 | Loss --> 1.852 | Grad_l2 --> 0.303 | Weights_l2 --> 9099.853 | Lr --> 0.003 | Seconds_per_step --> 3.362 | 
+[2024-08-11 13:08:06,024][Main][INFO] - [train] Step 51350 out of 80000 | Loss --> 1.849 | Grad_l2 --> 0.304 | Weights_l2 --> 9099.783 | Lr --> 0.003 | Seconds_per_step --> 3.351 | 
+[2024-08-11 13:10:53,915][Main][INFO] - [train] Step 51400 out of 80000 | Loss --> 1.839 | Grad_l2 --> 0.304 | Weights_l2 --> 9099.716 | Lr --> 0.003 | Seconds_per_step --> 3.358 | 
+[2024-08-11 13:13:41,963][Main][INFO] - [train] Step 51450 out of 80000 | Loss --> 1.845 | Grad_l2 --> 0.306 | Weights_l2 --> 9099.653 | Lr --> 0.003 | Seconds_per_step --> 3.361 | 
+[2024-08-11 13:16:31,982][Main][INFO] - [train] Step 51500 out of 80000 | Loss --> 1.839 | Grad_l2 --> 0.302 | Weights_l2 --> 9099.578 | Lr --> 0.003 | Seconds_per_step --> 3.400 | 
+[2024-08-11 13:19:21,318][Main][INFO] - [train] Step 51550 out of 80000 | Loss --> 1.837 | Grad_l2 --> 0.303 | Weights_l2 --> 9099.520 | Lr --> 0.003 | Seconds_per_step --> 3.387 | 
+[2024-08-11 13:22:09,644][Main][INFO] - [train] Step 51600 out of 80000 | Loss --> 1.833 | Grad_l2 --> 0.305 | Weights_l2 --> 9099.446 | Lr --> 0.003 | Seconds_per_step --> 3.367 | 
+[2024-08-11 13:25:06,187][Main][INFO] - [train] Step 51650 out of 80000 | Loss --> 1.842 | Grad_l2 --> 0.303 | Weights_l2 --> 9099.375 | Lr --> 0.003 | Seconds_per_step --> 3.531 | 
+[2024-08-11 13:27:55,296][Main][INFO] - [train] Step 51700 out of 80000 | Loss --> 1.824 | Grad_l2 --> 0.304 | Weights_l2 --> 9099.306 | Lr --> 0.003 | Seconds_per_step --> 3.382 | 
+[2024-08-11 13:30:48,050][Main][INFO] - [train] Step 51750 out of 80000 | Loss --> 1.834 | Grad_l2 --> 0.304 | Weights_l2 --> 9099.235 | Lr --> 0.003 | Seconds_per_step --> 3.455 | 
+[2024-08-11 13:33:36,637][Main][INFO] - [train] Step 51800 out of 80000 | Loss --> 1.829 | Grad_l2 --> 0.303 | Weights_l2 --> 9099.164 | Lr --> 0.003 | Seconds_per_step --> 3.372 | 
+[2024-08-11 13:36:25,148][Main][INFO] - [train] Step 51850 out of 80000 | Loss --> 1.831 | Grad_l2 --> 0.306 | Weights_l2 --> 9099.098 | Lr --> 0.003 | Seconds_per_step --> 3.370 | 
+[2024-08-11 13:39:14,286][Main][INFO] - [train] Step 51900 out of 80000 | Loss --> 1.828 | Grad_l2 --> 0.304 | Weights_l2 --> 9099.024 | Lr --> 0.003 | Seconds_per_step --> 3.383 | 
+[2024-08-11 13:42:02,662][Main][INFO] - [train] Step 51950 out of 80000 | Loss --> 1.828 | Grad_l2 --> 0.305 | Weights_l2 --> 9098.956 | Lr --> 0.003 | Seconds_per_step --> 3.368 | 
+[2024-08-11 13:44:51,092][Main][INFO] - [train] Step 52000 out of 80000 | Loss --> 1.826 | Grad_l2 --> 0.305 | Weights_l2 --> 9098.885 | Lr --> 0.003 | Seconds_per_step --> 3.369 | 
+[2024-08-11 13:47:40,032][Main][INFO] - [train] Step 52050 out of 80000 | Loss --> 1.822 | Grad_l2 --> 0.302 | Weights_l2 --> 9098.823 | Lr --> 0.003 | Seconds_per_step --> 3.379 | 
+[2024-08-11 13:50:29,006][Main][INFO] - [train] Step 52100 out of 80000 | Loss --> 1.825 | Grad_l2 --> 0.305 | Weights_l2 --> 9098.748 | Lr --> 0.003 | Seconds_per_step --> 3.379 | 
+[2024-08-11 13:53:17,565][Main][INFO] - [train] Step 52150 out of 80000 | Loss --> 1.823 | Grad_l2 --> 0.302 | Weights_l2 --> 9098.682 | Lr --> 0.003 | Seconds_per_step --> 3.371 | 
+[2024-08-11 13:56:06,216][Main][INFO] - [train] Step 52200 out of 80000 | Loss --> 1.820 | Grad_l2 --> 0.303 | Weights_l2 --> 9098.606 | Lr --> 0.003 | Seconds_per_step --> 3.373 | 
+[2024-08-11 13:58:54,102][Main][INFO] - [train] Step 52250 out of 80000 | Loss --> 1.816 | Grad_l2 --> 0.303 | Weights_l2 --> 9098.532 | Lr --> 0.003 | Seconds_per_step --> 3.358 | 
+[2024-08-11 14:01:43,559][Main][INFO] - [train] Step 52300 out of 80000 | Loss --> 1.830 | Grad_l2 --> 0.302 | Weights_l2 --> 9098.464 | Lr --> 0.003 | Seconds_per_step --> 3.389 | 
+[2024-08-11 14:04:32,277][Main][INFO] - [train] Step 52350 out of 80000 | Loss --> 1.814 | Grad_l2 --> 0.302 | Weights_l2 --> 9098.392 | Lr --> 0.003 | Seconds_per_step --> 3.374 | 
+[2024-08-11 14:07:20,639][Main][INFO] - [train] Step 52400 out of 80000 | Loss --> 1.815 | Grad_l2 --> 0.306 | Weights_l2 --> 9098.315 | Lr --> 0.003 | Seconds_per_step --> 3.367 | 
+[2024-08-11 14:10:08,465][Main][INFO] - [train] Step 52450 out of 80000 | Loss --> 1.813 | Grad_l2 --> 0.304 | Weights_l2 --> 9098.238 | Lr --> 0.003 | Seconds_per_step --> 3.357 | 
+[2024-08-11 14:12:57,599][Main][INFO] - [train] Step 52500 out of 80000 | Loss --> 1.823 | Grad_l2 --> 0.304 | Weights_l2 --> 9098.172 | Lr --> 0.003 | Seconds_per_step --> 3.383 | 
+[2024-08-11 14:15:45,660][Main][INFO] - [train] Step 52550 out of 80000 | Loss --> 1.813 | Grad_l2 --> 0.306 | Weights_l2 --> 9098.102 | Lr --> 0.003 | Seconds_per_step --> 3.361 | 
+[2024-08-11 14:18:34,118][Main][INFO] - [train] Step 52600 out of 80000 | Loss --> 1.820 | Grad_l2 --> 0.305 | Weights_l2 --> 9098.032 | Lr --> 0.003 | Seconds_per_step --> 3.369 | 
+[2024-08-11 14:21:22,137][Main][INFO] - [train] Step 52650 out of 80000 | Loss --> 1.810 | Grad_l2 --> 0.306 | Weights_l2 --> 9097.958 | Lr --> 0.003 | Seconds_per_step --> 3.360 | 
+[2024-08-11 14:24:11,153][Main][INFO] - [train] Step 52700 out of 80000 | Loss --> 1.819 | Grad_l2 --> 0.306 | Weights_l2 --> 9097.883 | Lr --> 0.003 | Seconds_per_step --> 3.380 | 
+[2024-08-11 14:26:59,325][Main][INFO] - [train] Step 52750 out of 80000 | Loss --> 1.819 | Grad_l2 --> 0.305 | Weights_l2 --> 9097.801 | Lr --> 0.003 | Seconds_per_step --> 3.363 | 
+[2024-08-11 14:29:48,152][Main][INFO] - [train] Step 52800 out of 80000 | Loss --> 1.818 | Grad_l2 --> 0.304 | Weights_l2 --> 9097.728 | Lr --> 0.003 | Seconds_per_step --> 3.377 | 
+[2024-08-11 14:32:36,719][Main][INFO] - [train] Step 52850 out of 80000 | Loss --> 1.816 | Grad_l2 --> 0.302 | Weights_l2 --> 9097.651 | Lr --> 0.003 | Seconds_per_step --> 3.371 | 
+[2024-08-11 14:35:26,695][Main][INFO] - [train] Step 52900 out of 80000 | Loss --> 1.821 | Grad_l2 --> 0.304 | Weights_l2 --> 9097.564 | Lr --> 0.003 | Seconds_per_step --> 3.400 | 
+[2024-08-11 14:38:15,366][Main][INFO] - [train] Step 52950 out of 80000 | Loss --> 1.830 | Grad_l2 --> 0.306 | Weights_l2 --> 9097.480 | Lr --> 0.003 | Seconds_per_step --> 3.373 | 
+[2024-08-11 14:41:03,151][Main][INFO] - [train] Step 53000 out of 80000 | Loss --> 1.816 | Grad_l2 --> 0.305 | Weights_l2 --> 9097.407 | Lr --> 0.003 | Seconds_per_step --> 3.356 | 
+[2024-08-11 14:43:51,226][Main][INFO] - [train] Step 53050 out of 80000 | Loss --> 1.822 | Grad_l2 --> 0.303 | Weights_l2 --> 9097.331 | Lr --> 0.003 | Seconds_per_step --> 3.361 | 

checkpoints/training_metrics.csv

@@ -1003,3 +1003,60 @@ timestamp,step,loss,grad_l2,weights_l2,lr,seconds_per_step
 "2024-08-11 11:57:38,118",50100,1.853,0.305,9101.232,0.003,3.388
 "2024-08-11 12:00:27,137",50150,1.858,0.305,9101.189,0.003,3.38
 "2024-08-11 12:03:16,714",50200,1.853,0.304,9101.138,0.003,3.392
+"2024-08-11 12:06:06,587",50250,1.863,0.303,9101.077,0.003,3.397
+"2024-08-11 12:08:54,840",50300,1.859,0.308,9101.026,0.003,3.365
+"2024-08-11 12:11:43,859",50350,1.867,0.306,9100.972,0.003,3.38
+"2024-08-11 12:14:31,964",50400,1.858,0.307,9100.919,0.003,3.362
+"2024-08-11 12:17:20,154",50450,1.865,0.306,9100.876,0.003,3.364
+"2024-08-11 12:20:08,016",50500,1.856,0.304,9100.82,0.003,3.357
+"2024-08-11 12:22:56,654",50550,1.859,0.306,9100.766,0.003,3.373
+"2024-08-11 12:25:46,183",50600,1.859,0.304,9100.712,0.003,3.391
+"2024-08-11 12:28:36,862",50650,1.866,0.304,9100.66,0.003,3.414
+"2024-08-11 12:31:26,475",50700,1.857,0.302,9100.605,0.003,3.392
+"2024-08-11 12:34:15,202",50750,1.856,0.308,9100.546,0.003,3.375
+"2024-08-11 12:37:04,603",50800,1.857,0.304,9100.481,0.003,3.388
+"2024-08-11 12:39:53,849",50850,1.851,0.304,9100.431,0.003,3.385
+"2024-08-11 12:42:42,323",50900,1.856,0.304,9100.369,0.003,3.369
+"2024-08-11 12:45:30,712",50950,1.863,0.304,9100.297,0.003,3.368
+"2024-08-11 12:48:19,531",51000,1.856,0.305,9100.235,0.003,3.376
+"2024-08-11 12:51:12,887",51050,1.851,0.302,9100.189,0.003,3.467
+"2024-08-11 12:54:01,929",51100,1.847,0.306,9100.124,0.003,3.381
+"2024-08-11 12:56:51,344",51150,1.841,0.305,9100.062,0.003,3.388
+"2024-08-11 12:59:41,233",51200,1.85,0.305,9099.993,0.003,3.398
+"2024-08-11 13:02:30,398",51250,1.85,0.302,9099.923,0.003,3.383
+"2024-08-11 13:05:18,484",51300,1.852,0.303,9099.853,0.003,3.362
+"2024-08-11 13:08:06,024",51350,1.849,0.304,9099.783,0.003,3.351
+"2024-08-11 13:10:53,915",51400,1.839,0.304,9099.716,0.003,3.358
+"2024-08-11 13:13:41,963",51450,1.845,0.306,9099.653,0.003,3.361
+"2024-08-11 13:16:31,982",51500,1.839,0.302,9099.578,0.003,3.4
+"2024-08-11 13:19:21,318",51550,1.837,0.303,9099.52,0.003,3.387
+"2024-08-11 13:22:09,644",51600,1.833,0.305,9099.446,0.003,3.367
+"2024-08-11 13:25:06,187",51650,1.842,0.303,9099.375,0.003,3.531
+"2024-08-11 13:27:55,296",51700,1.824,0.304,9099.306,0.003,3.382
+"2024-08-11 13:30:48,050",51750,1.834,0.304,9099.235,0.003,3.455
+"2024-08-11 13:33:36,637",51800,1.829,0.303,9099.164,0.003,3.372
+"2024-08-11 13:36:25,148",51850,1.831,0.306,9099.098,0.003,3.37
+"2024-08-11 13:39:14,286",51900,1.828,0.304,9099.024,0.003,3.383
+"2024-08-11 13:42:02,662",51950,1.828,0.305,9098.956,0.003,3.368
+"2024-08-11 13:44:51,092",52000,1.826,0.305,9098.885,0.003,3.369
+"2024-08-11 13:47:40,032",52050,1.822,0.302,9098.823,0.003,3.379
+"2024-08-11 13:50:29,006",52100,1.825,0.305,9098.748,0.003,3.379
+"2024-08-11 13:53:17,565",52150,1.823,0.302,9098.682,0.003,3.371
+"2024-08-11 13:56:06,216",52200,1.82,0.303,9098.606,0.003,3.373
+"2024-08-11 13:58:54,102",52250,1.816,0.303,9098.532,0.003,3.358
+"2024-08-11 14:01:43,559",52300,1.83,0.302,9098.464,0.003,3.389
+"2024-08-11 14:04:32,277",52350,1.814,0.302,9098.392,0.003,3.374
+"2024-08-11 14:07:20,639",52400,1.815,0.306,9098.315,0.003,3.367
+"2024-08-11 14:10:08,465",52450,1.813,0.304,9098.238,0.003,3.357
+"2024-08-11 14:12:57,599",52500,1.823,0.304,9098.172,0.003,3.383
+"2024-08-11 14:15:45,660",52550,1.813,0.306,9098.102,0.003,3.361
+"2024-08-11 14:18:34,118",52600,1.82,0.305,9098.032,0.003,3.369
+"2024-08-11 14:21:22,137",52650,1.81,0.306,9097.958,0.003,3.36
+"2024-08-11 14:24:11,153",52700,1.819,0.306,9097.883,0.003,3.38
+"2024-08-11 14:26:59,325",52750,1.819,0.305,9097.801,0.003,3.363
+"2024-08-11 14:29:48,152",52800,1.818,0.304,9097.728,0.003,3.377
+"2024-08-11 14:32:36,719",52850,1.816,0.302,9097.651,0.003,3.371
+"2024-08-11 14:35:26,695",52900,1.821,0.304,9097.564,0.003,3.4
+"2024-08-11 14:38:15,366",52950,1.83,0.306,9097.48,0.003,3.373
+"2024-08-11 14:41:03,151",53000,1.816,0.305,9097.407,0.003,3.356
+"2024-08-11 14:43:51,226",53050,1.822,0.303,9097.331,0.003,3.361