Training in progress, step 1000

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d7f84b30ad1e26b72493f2e487a84b8fb077327a611d56fcd0605d78146fa822
+oid sha256:9a7667b4f7fb01bc0f00d3a07e35846f784c9d9bf8171e61c789861b33c4987f
 size 3141646744

runs/Apr24_16-42-31_ip-26-0-162-233/events.out.tfevents.1713977002.ip-26-0-162-233.1854033.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d99d51bb7fcd506f76273107b7f54b0a07695a2cf620317840bf9823aa458c38
-size 9086
+oid sha256:283b4ce5861b6a32871a30e6646ee3c820d4f3b45c93457f52525e6cba33a9a4
+size 13306

wandb/run-20240424_164324-xfbnm7qo/files/output.log

@@ -520,3 +520,482 @@
 
 
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  3%|██                                                                           | 550/20000 [17:12<9:50:23,  1.82s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  3%|██▏                                                                          | 575/20000 [17:57<9:51:03,  1.83s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  3%|██▎                                                                          | 600/20000 [18:43<9:49:40,  1.82s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  3%|██▍                                                                          | 625/20000 [19:28<9:46:32,  1.82s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  3%|██▌                                                                          | 650/20000 [20:14<9:44:36,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  3%|██▌                                                                          | 675/20000 [20:59<9:45:00,  1.82s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  4%|██▋                                                                          | 700/20000 [21:45<9:46:59,  1.82s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  4%|██▊                                                                          | 724/20000 [22:28<9:44:44,  1.82s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  4%|██▉                                                                          | 750/20000 [23:16<9:44:11,  1.82s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  4%|██▉                                                                          | 775/20000 [24:01<9:42:52,  1.82s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  4%|███                                                                         | 800/20000 [24:47<10:00:33,  1.88s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  4%|███▏                                                                         | 824/20000 [25:30<9:42:02,  1.82s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  4%|███▎                                                                         | 850/20000 [26:18<9:41:27,  1.82s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  4%|███▎                                                                         | 875/20000 [27:03<9:40:29,  1.82s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  4%|███▍                                                                         | 900/20000 [27:49<9:39:54,  1.82s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  5%|███▌                                                                         | 924/20000 [28:32<9:40:41,  1.83s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  5%|███▋                                                                         | 950/20000 [29:20<9:37:26,  1.82s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  5%|███▊                                                                         | 975/20000 [30:05<9:34:44,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  5%|███▊                                                                        | 1000/20000 [30:51<9:35:10,  1.82s/it][INFO|trainer.py:3304] 2024-04-24 17:14:21,632 >> Saving model checkpoint to ./checkpoint-1000
+[INFO|configuration_utils.py:471] 2024-04-24 17:14:21,635 >> Configuration saved in ./checkpoint-1000/config.json
+[INFO|configuration_utils.py:697] 2024-04-24 17:14:21,638 >> Configuration saved in ./checkpoint-1000/generation_config.json
+{'loss': 1.5546, 'grad_norm': 1.2265625, 'learning_rate': 9.743589743589744e-05, 'epoch': 0.25}
+[INFO|modeling_utils.py:2590] 2024-04-24 17:14:25,965 >> Model weights saved in ./checkpoint-1000/model.safetensors
+[INFO|tokenization_utils_base.py:2488] 2024-04-24 17:14:25,975 >> tokenizer config file saved in ./checkpoint-1000/tokenizer_config.json
+[INFO|tokenization_utils_base.py:2497] 2024-04-24 17:14:25,977 >> Special tokens file saved in ./checkpoint-1000/special_tokens_map.json
+[INFO|tokenization_utils_base.py:2488] 2024-04-24 17:14:35,995 >> tokenizer config file saved in ./tokenizer_config.json
+[INFO|tokenization_utils_base.py:2497] 2024-04-24 17:14:35,997 >> Special tokens file saved in ./special_tokens_map.json
+/fsx/sanchit/miniconda3/envs/venv/lib/python3.11/site-packages/torch/utils/checkpoint.py:144: UserWarning: Tensor arguments, excluding CPU tensors, are detected on at least two types of devices. Device state will only be saved for devices of a single device type, and the remaining devices will be ignored. Consequently, if any checkpointed functions involve randomness, this may result in incorrect gradients. (Note that if CUDA devices are among the devices detected, it will be prioritized; otherwise, the first device encountered will be selected.)
+  warnings.warn(
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  5%|███▉                                                                        | 1026/20000 [31:52<9:35:24,  1.82s/it]
+
+
+
+

wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 2.0215, "train/grad_norm": 4.125, "train/learning_rate": 9.987179487179488e-05, "train/epoch": 0.13, "train/global_step": 525, "_timestamp": 1713977997.0387745, "_runtime": 992.4967684745789, "_step": 21}
+{"train/loss": 1.5537, "train/grad_norm": 1.171875, "train/learning_rate": 9.730769230769232e-05, "train/epoch": 0.25, "train/global_step": 1025, "_timestamp": 1713978921.4080203, "_runtime": 1916.8660142421722, "_step": 41}