Add files using upload-large-folder tool

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+541358.err filter=lfs diff=lfs merge=lfs -text
+541360.err filter=lfs diff=lfs merge=lfs -text
+541285.err filter=lfs diff=lfs merge=lfs -text

541270.err

@@ -0,0 +1,7 @@
+[NbConvertApp] Converting notebook HCP_downstream_raw_flatmaps.ipynb to python
+[NbConvertApp] Writing 36448 bytes to HCP_downstream_raw_flatmaps.py
+Traceback (most recent call last):
+  File "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_raw_flatmaps.py", line 87, in <module>
+    utils.seed_everything(seed)
+                          ^^^^
+NameError: name 'seed' is not defined

541270.out

@@ -0,0 +1,5 @@
+NUM_GPUS=1
+MASTER_ADDR=ip-10-0-133-32
+MASTER_PORT=11200
+WORLD_SIZE=1
+PID of this process = 1825497

541272.err

@@ -0,0 +1,7 @@
+[NbConvertApp] Converting notebook HCP_downstream_raw_flatmaps.ipynb to python
+[NbConvertApp] Writing 36446 bytes to HCP_downstream_raw_flatmaps.py
+Traceback (most recent call last):
+  File "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_raw_flatmaps.py", line 737, in <module>
+    optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate, weight_decay=weight_decay)
+                                                         ^^^^^^^^^^^^^
+NameError: name 'learning_rate' is not defined

541272.out

@@ -0,0 +1,8 @@
+NUM_GPUS=1
+MASTER_ADDR=ip-10-0-133-32
+MASTER_PORT=12606
+WORLD_SIZE=1
+PID of this process = 1826825
+------ ARGS ------- 
+ Namespace(model_suffix='beta', hcp_flat_path='/weka/proj-medarc/shared/HCP-Flat', batch_size=128, wandb_log=True, num_epochs=20, lr_scheduler_type='cycle', save_ckpt=False, seed=42, max_lr=0.1, target='sex', num_workers=15, weight_decay=1e-05)
+Input dimension: 737280

541275.err

@@ -0,0 +1,7 @@
+[NbConvertApp] Converting notebook HCP_downstream_raw_flatmaps.ipynb to python
+[NbConvertApp] Writing 36336 bytes to HCP_downstream_raw_flatmaps.py
+Traceback (most recent call last):
+  File "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_raw_flatmaps.py", line 727, in <module>
+    num_iterations_per_epoch = math.ceil(flatmaps_train/batch_size)
+                                         ~~~~~~~~~~~~~~^~~~~~~~~~~
+TypeError: unsupported operand type(s) for /: 'Dataset' and 'int'

541275.out

@@ -0,0 +1,8 @@
+NUM_GPUS=1
+MASTER_ADDR=ip-10-0-136-246
+MASTER_PORT=17060
+WORLD_SIZE=1
+PID of this process = 564369
+------ ARGS ------- 
+ Namespace(model_suffix='beta', hcp_flat_path='/weka/proj-medarc/shared/HCP-Flat', batch_size=128, wandb_log=True, num_epochs=20, lr_scheduler_type='cycle', save_ckpt=False, seed=42, max_lr=0.1, target='sex', num_workers=15, weight_decay=1e-05)
+Input dimension: 737280

541276.out

@@ -0,0 +1,194 @@
+NUM_GPUS=1
+MASTER_ADDR=ip-10-0-136-246
+MASTER_PORT=16668
+WORLD_SIZE=1
+PID of this process = 565724
+------ ARGS ------- 
+ Namespace(model_suffix='beta', hcp_flat_path='/weka/proj-medarc/shared/HCP-Flat', batch_size=128, wandb_log=True, num_epochs=20, lr_scheduler_type='cycle', save_ckpt=False, seed=42, max_lr=0.1, target='sex', num_workers=15, weight_decay=1e-05)
+Input dimension: 737280
+total_steps 17400
+wandb_config:
+ {'model_name': 'HCPflat_raw_sex', 'batch_size': 128, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42, 'lr_scheduler_type': 'cycle', 'save_ckpt': False, 'max_lr': 0.1, 'target': 'sex', 'num_workers': 15}
+wandb_id: HCPflat_raw_beta_sex_83810
+Step [100/870] - Training Loss: 24.6087 - Training Accuracy: 52.39%
+Step [200/870] - Training Loss: 28.9030 - Training Accuracy: 52.85%
+Step [300/870] - Training Loss: 50.2400 - Training Accuracy: 53.05%
+Step [400/870] - Training Loss: 74.8264 - Training Accuracy: 53.27%
+Step [500/870] - Training Loss: 91.1850 - Training Accuracy: 53.40%
+Step [600/870] - Training Loss: 165.0463 - Training Accuracy: 53.63%
+Step [700/870] - Training Loss: 201.5126 - Training Accuracy: 53.66%
+Step [800/870] - Training Loss: 209.5273 - Training Accuracy: 53.74%
+Epoch [1/20] - Training Loss: 110.8142, Training Accuracy: 53.88% - Validation Loss: 296.1424, Validation Accuracy: 53.70%
+Step [100/870] - Training Loss: 298.0217 - Training Accuracy: 66.52%
+Step [200/870] - Training Loss: 265.3320 - Training Accuracy: 65.68%
+Step [300/870] - Training Loss: 293.1298 - Training Accuracy: 64.77%
+Step [400/870] - Training Loss: 553.0426 - Training Accuracy: 64.16%
+Step [500/870] - Training Loss: 594.5250 - Training Accuracy: 63.51%
+Step [600/870] - Training Loss: 708.0252 - Training Accuracy: 62.87%
+Step [700/870] - Training Loss: 722.7825 - Training Accuracy: 62.39%
+Step [800/870] - Training Loss: 798.4144 - Training Accuracy: 61.97%
+Epoch [2/20] - Training Loss: 449.1676, Training Accuracy: 61.70% - Validation Loss: 808.8942, Validation Accuracy: 54.41%
+Step [100/870] - Training Loss: 396.8062 - Training Accuracy: 75.48%
+Step [200/870] - Training Loss: 465.8516 - Training Accuracy: 75.21%
+Step [300/870] - Training Loss: 334.3605 - Training Accuracy: 75.27%
+Step [400/870] - Training Loss: 362.5482 - Training Accuracy: 74.79%
+Step [500/870] - Training Loss: 458.4806 - Training Accuracy: 74.32%
+Step [600/870] - Training Loss: 336.7921 - Training Accuracy: 73.79%
+Step [700/870] - Training Loss: 595.4280 - Training Accuracy: 73.40%
+Step [800/870] - Training Loss: 591.4528 - Training Accuracy: 73.04%
+Epoch [3/20] - Training Loss: 434.5445, Training Accuracy: 72.76% - Validation Loss: 1042.0977, Validation Accuracy: 54.73%
+Step [100/870] - Training Loss: 270.8356 - Training Accuracy: 83.11%
+Step [200/870] - Training Loss: 361.4072 - Training Accuracy: 83.15%
+Step [300/870] - Training Loss: 275.5848 - Training Accuracy: 82.82%
+Step [400/870] - Training Loss: 307.0319 - Training Accuracy: 82.44%
+Step [500/870] - Training Loss: 326.9714 - Training Accuracy: 82.14%
+Step [600/870] - Training Loss: 271.0794 - Training Accuracy: 81.70%
+Step [700/870] - Training Loss: 260.8827 - Training Accuracy: 81.37%
+Step [800/870] - Training Loss: 419.5749 - Training Accuracy: 81.08%
+Epoch [4/20] - Training Loss: 296.3454, Training Accuracy: 80.79% - Validation Loss: 1187.6379, Validation Accuracy: 55.25%
+Step [100/870] - Training Loss: 214.9724 - Training Accuracy: 87.92%
+Step [200/870] - Training Loss: 77.4744 - Training Accuracy: 87.77%
+Step [300/870] - Training Loss: 149.2222 - Training Accuracy: 87.47%
+Step [400/870] - Training Loss: 141.0663 - Training Accuracy: 87.16%
+Step [500/870] - Training Loss: 231.0289 - Training Accuracy: 86.83%
+Step [600/870] - Training Loss: 186.0840 - Training Accuracy: 86.38%
+Step [700/870] - Training Loss: 163.8004 - Training Accuracy: 85.99%
+Step [800/870] - Training Loss: 304.4012 - Training Accuracy: 85.70%
+Epoch [5/20] - Training Loss: 211.2076, Training Accuracy: 85.50% - Validation Loss: 1311.0324, Validation Accuracy: 55.02%
+Step [100/870] - Training Loss: 112.8653 - Training Accuracy: 90.42%
+Step [200/870] - Training Loss: 182.0056 - Training Accuracy: 90.31%
+Step [300/870] - Training Loss: 151.2417 - Training Accuracy: 90.17%
+Step [400/870] - Training Loss: 174.8410 - Training Accuracy: 89.76%
+Step [500/870] - Training Loss: 164.9281 - Training Accuracy: 89.42%
+Step [600/870] - Training Loss: 176.1206 - Training Accuracy: 89.19%
+Step [700/870] - Training Loss: 189.1104 - Training Accuracy: 88.92%
+Step [800/870] - Training Loss: 164.0583 - Training Accuracy: 88.64%
+Epoch [6/20] - Training Loss: 161.3509, Training Accuracy: 88.49% - Validation Loss: 1458.8603, Validation Accuracy: 54.97%
+Step [100/870] - Training Loss: 96.9147 - Training Accuracy: 91.70%
+Step [200/870] - Training Loss: 89.6436 - Training Accuracy: 91.68%
+Step [300/870] - Training Loss: 88.9899 - Training Accuracy: 91.55%
+Step [400/870] - Training Loss: 90.7214 - Training Accuracy: 91.36%
+Step [500/870] - Training Loss: 246.9420 - Training Accuracy: 91.10%
+Step [600/870] - Training Loss: 143.6372 - Training Accuracy: 90.95%
+Step [700/870] - Training Loss: 132.4662 - Training Accuracy: 90.78%
+Step [800/870] - Training Loss: 199.1868 - Training Accuracy: 90.55%
+Epoch [7/20] - Training Loss: 130.6820, Training Accuracy: 90.41% - Validation Loss: 1515.6320, Validation Accuracy: 55.45%
+Step [100/870] - Training Loss: 40.2281 - Training Accuracy: 93.41%
+Step [200/870] - Training Loss: 31.9451 - Training Accuracy: 93.21%
+Step [300/870] - Training Loss: 51.2280 - Training Accuracy: 93.23%
+Step [400/870] - Training Loss: 100.9511 - Training Accuracy: 93.02%
+Step [500/870] - Training Loss: 103.3127 - Training Accuracy: 92.90%
+Step [600/870] - Training Loss: 152.1203 - Training Accuracy: 92.78%
+Step [700/870] - Training Loss: 108.2650 - Training Accuracy: 92.67%
+Step [800/870] - Training Loss: 93.6054 - Training Accuracy: 92.52%
+Epoch [8/20] - Training Loss: 96.2420, Training Accuracy: 92.41% - Validation Loss: 1629.8896, Validation Accuracy: 55.26%
+Step [100/870] - Training Loss: 48.9615 - Training Accuracy: 95.19%
+Step [200/870] - Training Loss: 37.2198 - Training Accuracy: 95.09%
+Step [300/870] - Training Loss: 57.5891 - Training Accuracy: 94.78%
+Step [400/870] - Training Loss: 116.6951 - Training Accuracy: 94.65%
+Step [500/870] - Training Loss: 106.4395 - Training Accuracy: 94.49%
+Step [600/870] - Training Loss: 67.2050 - Training Accuracy: 94.33%
+Step [700/870] - Training Loss: 29.4207 - Training Accuracy: 94.26%
+Step [800/870] - Training Loss: 19.4606 - Training Accuracy: 94.18%
+Epoch [9/20] - Training Loss: 70.5700, Training Accuracy: 94.06% - Validation Loss: 1667.6055, Validation Accuracy: 54.94%
+Step [100/870] - Training Loss: 133.2186 - Training Accuracy: 95.77%
+Step [200/870] - Training Loss: 39.1579 - Training Accuracy: 96.05%
+Step [300/870] - Training Loss: 19.6516 - Training Accuracy: 95.85%
+Step [400/870] - Training Loss: 14.1961 - Training Accuracy: 95.73%
+Step [500/870] - Training Loss: 69.0657 - Training Accuracy: 95.66%
+Step [600/870] - Training Loss: 86.5776 - Training Accuracy: 95.54%
+Step [700/870] - Training Loss: 40.0283 - Training Accuracy: 95.48%
+Step [800/870] - Training Loss: 74.8730 - Training Accuracy: 95.35%
+Epoch [10/20] - Training Loss: 51.8441, Training Accuracy: 95.28% - Validation Loss: 1732.3389, Validation Accuracy: 55.58%
+Step [100/870] - Training Loss: 15.9613 - Training Accuracy: 97.03%
+Step [200/870] - Training Loss: 17.7464 - Training Accuracy: 96.98%
+Step [300/870] - Training Loss: 22.3283 - Training Accuracy: 96.74%
+Step [400/870] - Training Loss: 38.1730 - Training Accuracy: 96.69%
+Step [500/870] - Training Loss: 4.6681 - Training Accuracy: 96.59%
+Step [600/870] - Training Loss: 28.4868 - Training Accuracy: 96.55%
+Step [700/870] - Training Loss: 55.0123 - Training Accuracy: 96.50%
+Step [800/870] - Training Loss: 26.6057 - Training Accuracy: 96.43%
+Epoch [11/20] - Training Loss: 36.3423, Training Accuracy: 96.41% - Validation Loss: 1742.7579, Validation Accuracy: 55.19%
+Step [100/870] - Training Loss: 8.1488 - Training Accuracy: 97.70%
+Step [200/870] - Training Loss: 18.2396 - Training Accuracy: 97.66%
+Step [300/870] - Training Loss: 0.0000 - Training Accuracy: 97.64%
+Step [400/870] - Training Loss: 2.1231 - Training Accuracy: 97.49%
+Step [500/870] - Training Loss: 8.7330 - Training Accuracy: 97.50%
+Step [600/870] - Training Loss: 15.5849 - Training Accuracy: 97.42%
+Step [700/870] - Training Loss: 5.5085 - Training Accuracy: 97.39%
+Step [800/870] - Training Loss: 93.1239 - Training Accuracy: 97.39%
+Epoch [12/20] - Training Loss: 23.7009, Training Accuracy: 97.35% - Validation Loss: 1784.1253, Validation Accuracy: 55.40%
+Step [100/870] - Training Loss: 4.8382 - Training Accuracy: 98.20%
+Step [200/870] - Training Loss: 25.5308 - Training Accuracy: 98.27%
+Step [300/870] - Training Loss: 11.4365 - Training Accuracy: 98.38%
+Step [400/870] - Training Loss: 0.1192 - Training Accuracy: 98.33%
+Step [500/870] - Training Loss: 13.1149 - Training Accuracy: 98.35%
+Step [600/870] - Training Loss: 0.7187 - Training Accuracy: 98.30%
+Step [700/870] - Training Loss: 18.9833 - Training Accuracy: 98.26%
+Step [800/870] - Training Loss: 10.3944 - Training Accuracy: 98.25%
+Epoch [13/20] - Training Loss: 12.9698, Training Accuracy: 98.21% - Validation Loss: 1779.4520, Validation Accuracy: 55.43%
+Step [100/870] - Training Loss: 0.2771 - Training Accuracy: 98.88%
+Step [200/870] - Training Loss: 6.9764 - Training Accuracy: 98.95%
+Step [300/870] - Training Loss: 6.0478 - Training Accuracy: 98.99%
+Step [400/870] - Training Loss: 7.7897 - Training Accuracy: 98.92%
+Step [500/870] - Training Loss: 0.0729 - Training Accuracy: 98.92%
+Step [600/870] - Training Loss: 24.3455 - Training Accuracy: 98.91%
+Step [700/870] - Training Loss: 3.5273 - Training Accuracy: 98.91%
+Step [800/870] - Training Loss: 1.3470 - Training Accuracy: 98.88%
+Epoch [14/20] - Training Loss: 6.8809, Training Accuracy: 98.87% - Validation Loss: 1781.8475, Validation Accuracy: 55.26%
+Step [100/870] - Training Loss: 14.5071 - Training Accuracy: 99.46%
+Step [200/870] - Training Loss: 15.6453 - Training Accuracy: 99.30%
+Step [300/870] - Training Loss: 8.2637 - Training Accuracy: 99.29%
+Step [400/870] - Training Loss: 0.0000 - Training Accuracy: 99.34%
+Step [500/870] - Training Loss: 0.0000 - Training Accuracy: 99.35%
+Step [600/870] - Training Loss: 5.3401 - Training Accuracy: 99.34%
+Step [700/870] - Training Loss: 0.0000 - Training Accuracy: 99.30%
+Step [800/870] - Training Loss: 2.4279 - Training Accuracy: 99.29%
+Epoch [15/20] - Training Loss: 3.5025, Training Accuracy: 99.29% - Validation Loss: 1785.6069, Validation Accuracy: 55.18%
+Step [100/870] - Training Loss: 0.0000 - Training Accuracy: 99.59%
+Step [200/870] - Training Loss: 0.0000 - Training Accuracy: 99.59%
+Step [300/870] - Training Loss: 0.0000 - Training Accuracy: 99.60%
+Step [400/870] - Training Loss: 0.0000 - Training Accuracy: 99.59%
+Step [500/870] - Training Loss: 0.6290 - Training Accuracy: 99.60%
+Step [600/870] - Training Loss: 0.0002 - Training Accuracy: 99.60%
+Step [700/870] - Training Loss: 4.8578 - Training Accuracy: 99.60%
+Step [800/870] - Training Loss: 5.8444 - Training Accuracy: 99.59%
+Epoch [16/20] - Training Loss: 1.6734, Training Accuracy: 99.58% - Validation Loss: 1784.3434, Validation Accuracy: 55.04%
+Step [100/870] - Training Loss: 0.0000 - Training Accuracy: 99.81%
+Step [200/870] - Training Loss: 0.0000 - Training Accuracy: 99.80%
+Step [300/870] - Training Loss: 3.4523 - Training Accuracy: 99.80%
+Step [400/870] - Training Loss: 0.0000 - Training Accuracy: 99.79%
+Step [500/870] - Training Loss: 2.6484 - Training Accuracy: 99.80%
+Step [600/870] - Training Loss: 0.0000 - Training Accuracy: 99.80%
+Step [700/870] - Training Loss: 0.0000 - Training Accuracy: 99.80%
+Step [800/870] - Training Loss: 0.0000 - Training Accuracy: 99.80%
+Epoch [17/20] - Training Loss: 0.6116, Training Accuracy: 99.80% - Validation Loss: 1777.6393, Validation Accuracy: 55.25%
+Step [100/870] - Training Loss: 0.0000 - Training Accuracy: 99.90%
+Step [200/870] - Training Loss: 0.0000 - Training Accuracy: 99.89%
+Step [300/870] - Training Loss: 1.4993 - Training Accuracy: 99.89%
+Step [400/870] - Training Loss: 1.9943 - Training Accuracy: 99.89%
+Step [500/870] - Training Loss: 0.0000 - Training Accuracy: 99.90%
+Step [600/870] - Training Loss: 0.0000 - Training Accuracy: 99.90%
+Step [700/870] - Training Loss: 0.0000 - Training Accuracy: 99.90%
+Step [800/870] - Training Loss: 0.0000 - Training Accuracy: 99.91%
+Epoch [18/20] - Training Loss: 0.1591, Training Accuracy: 99.91% - Validation Loss: 1778.0985, Validation Accuracy: 55.21%
+Step [100/870] - Training Loss: 0.0000 - Training Accuracy: 99.95%
+Step [200/870] - Training Loss: 0.0000 - Training Accuracy: 99.97%
+Step [300/870] - Training Loss: 0.0000 - Training Accuracy: 99.96%
+Step [400/870] - Training Loss: 0.0000 - Training Accuracy: 99.97%
+Step [500/870] - Training Loss: 0.0000 - Training Accuracy: 99.97%
+Step [600/870] - Training Loss: 0.0000 - Training Accuracy: 99.97%
+Step [700/870] - Training Loss: 0.0000 - Training Accuracy: 99.98%
+Step [800/870] - Training Loss: 0.0000 - Training Accuracy: 99.98%
+Epoch [19/20] - Training Loss: 0.0197, Training Accuracy: 99.98% - Validation Loss: 1777.6055, Validation Accuracy: 55.31%
+Step [100/870] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [200/870] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [300/870] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [400/870] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [500/870] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [600/870] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [700/870] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [800/870] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Epoch [20/20] - Training Loss: 0.0008, Training Accuracy: 100.00% - Validation Loss: 1777.4007, Validation Accuracy: 55.26%
+wandb: 🚀 View run HCPflat_raw_beta_sex at: https://stability.wandb.io/ckadirt/fMRI-foundation-model/runs/HCPflat_raw_beta_sex_83810
+wandb: Find logs at: wandb/run-20241126_204427-HCPflat_raw_beta_sex_83810/logs

541280.err

@@ -0,0 +1,7 @@
+[NbConvertApp] Converting notebook HCP_downstream_finetune.ipynb to python
+[NbConvertApp] Writing 31620 bytes to HCP_downstream_finetune.py
+Traceback (most recent call last):
+  File "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py", line 72, in <module>
+    parser = argparse.ArgumentParser(description="Model Training Configuration")
+             ^^^^^^^^
+NameError: name 'argparse' is not defined

541280.out

@@ -0,0 +1,4 @@
+NUM_GPUS=1
+MASTER_ADDR=ip-10-0-133-32
+MASTER_PORT=13737
+WORLD_SIZE=1

541281.err

@@ -0,0 +1,20 @@
+[NbConvertApp] Converting notebook HCP_downstream_finetune.ipynb to python
+[NbConvertApp] Writing 31636 bytes to HCP_downstream_finetune.py
+Traceback (most recent call last):
+  File "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py", line 123, in <module>
+    parser.add_argument(
+  File "/usr/lib/python3.11/argparse.py", line 1485, in add_argument
+    return self._add_action(action)
+           ^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/lib/python3.11/argparse.py", line 1867, in _add_action
+    self._optionals._add_action(action)
+  File "/usr/lib/python3.11/argparse.py", line 1687, in _add_action
+    action = super(_ArgumentGroup, self)._add_action(action)
+             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/lib/python3.11/argparse.py", line 1499, in _add_action
+    self._check_conflict(action)
+  File "/usr/lib/python3.11/argparse.py", line 1636, in _check_conflict
+    conflict_handler(action, confl_optionals)
+  File "/usr/lib/python3.11/argparse.py", line 1645, in _handle_conflict_error
+    raise ArgumentError(action, message % conflict_string)
+argparse.ArgumentError: argument --wandb_log/--no-wandb_log: conflicting option strings: --wandb_log, --no-wandb_log

541281.out

@@ -0,0 +1,4 @@
+NUM_GPUS=1
+MASTER_ADDR=ip-10-0-133-32
+MASTER_PORT=13853
+WORLD_SIZE=1

541282.err

@@ -0,0 +1,54 @@
+[NbConvertApp] Converting notebook HCP_downstream_finetune.ipynb to python
+[NbConvertApp] Writing 31825 bytes to HCP_downstream_finetune.py
+/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py:658: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
+  state = torch.load(checkpoint_path)
+wandb: Using wandb-core as the SDK backend. Please refer to https://wandb.me/wandb-core for more information.
+wandb: Currently logged in as: ckadirt. Use `wandb login --relogin` to force relogin
+wandb: Tracking run with wandb version 0.18.3
+wandb: Run data is saved locally in /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_213704-HCPflat_large_gsrFalse__beta_sex_HCPFT_83810
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run HCPflat_large_gsrFalse__beta_sex_HCPFT
+wandb: ⭐️ View project at https://stability.wandb.io/ckadirt/fMRI-foundation-model
+wandb: 🚀 View run at https://stability.wandb.io/ckadirt/fMRI-foundation-model/runs/HCPflat_large_gsrFalse__beta_sex_HCPFT_83810
+
+Traceback (most recent call last):
+  File "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py", line 843, in <module>
+    outputs = model(images, gsr=gsr)  # Shape: [num_train_samples, num_classes]
+              ^^^^^^^^^^^^^^^^^^^^^^
+  File "/admin/home-ckadirt/foundation_env/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1553, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/admin/home-ckadirt/foundation_env/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1562, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py", line 696, in forward
+    x = self.mae_model(x, global_pool=global_pool, forward_features = True)
+        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/admin/home-ckadirt/foundation_env/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1553, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/admin/home-ckadirt/foundation_env/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1562, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/mae_utils/flat_models.py", line 753, in forward
+    x = blk(x)
+        ^^^^^^
+  File "/admin/home-ckadirt/foundation_env/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1553, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/admin/home-ckadirt/foundation_env/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1562, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/mae_utils/video_vit.py", line 166, in forward
+    x = x + self.drop_path(self.attn(self.norm1(x)))
+                           ^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/admin/home-ckadirt/foundation_env/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1553, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/admin/home-ckadirt/foundation_env/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1562, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/mae_utils/video_vit.py", line 114, in forward
+    attn = (q @ k.transpose(-2, -1)) * self.scale
+            ~~^~~~~~~~~~~~~~~~~~~~~
+torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 2.71 GiB. GPU 0 has a total capacity of 79.11 GiB of which 770.94 MiB is free. Including non-PyTorch memory, this process has 78.35 GiB memory in use. Of the allocated memory 74.92 GiB is allocated by PyTorch, and 2.77 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation.  See documentation for Memory Management  (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)

541282.out

@@ -0,0 +1,70 @@
+NUM_GPUS=1
+MASTER_ADDR=ip-10-0-133-32
+MASTER_PORT=14835
+WORLD_SIZE=1
+------ ARGS ------- 
+ Namespace(found_model_name='HCPflat_large_gsrFalse_', epoch_checkpoint='epoch99.pth', model_suffix='beta', hcp_flat_path='/weka/proj-medarc/shared/HCP-Flat', batch_size=32, wandb_log=True, num_epochs=20, lr_scheduler_type='cycle', save_ckpt=False, seed=42, max_lr=0.0003, target='sex', num_workers=15, weight_decay=0.001, global_pool=True)
+outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_
+Loaded config.yaml from ckpt folder /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_
+
+__CONFIG__
+base_lr = 0.001
+batch_size = 32
+ckpt_interval = 5
+ckpt_saving = True
+cls_embed = True
+contrastive_loss_weight = 1.0
+datasets_to_include = HCP
+decoder_embed_dim = 512
+grad_accumulation_steps = 1
+grad_clip = 1.0
+gsr = False
+hcp_flat_path = /weka/proj-medarc/shared/HCP-Flat
+mask_ratio = 0.75
+model_name = HCPflat_large_gsrFalse_
+no_qkv_bias = False
+norm_pix_loss = False
+nsd_flat_path = /weka/proj-medarc/shared/NSD-Flat
+num_epochs = 100
+num_frames = 16
+num_samples_per_epoch = 200000
+num_workers = 10
+patch_size = 16
+pct_masks_to_decode = 1
+plotting = True
+pred_t_dim = 8
+print_interval = 20
+probe_base_lr = 0.0003
+probe_batch_size = 8
+probe_num_epochs = 30
+probe_num_samples_per_epoch = 100000
+resume_from_ckpt = True
+seed = 42
+sep_pos_embed = True
+t_patch_size = 2
+test_num_samples_per_epoch = 50000
+test_set = False
+trunc_init = False
+use_contrastive_loss = False
+wandb_log = True
+
+
+WORLD_SIZE=1
+PID of this process = 1885623
+global_pool = True
+gsr = False
+Creating datasets
+Datasets ready
+img_size (144, 320) patch_size (16, 16) frames 16 t_patch_size 2
+model initialized
+latest_checkpoint: epoch99.pth
+
+Loaded checkpoint epoch99.pth from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_
+
+Input dimension: 1024
+total_steps 69580
+wandb_config:
+ {'model_name': 'HCPflat_large_gsrFalse__HCP_FT_sex', 'batch_size': 32, 'weight_decay': 0.001, 'num_epochs': 20, 'seed': 42, 'lr_scheduler_type': 'cycle', 'save_ckpt': False, 'max_lr': 0.0003, 'target': 'sex', 'num_workers': 15}
+wandb_id: HCPflat_large_gsrFalse__beta_sex_HCPFT_83810
+wandb: 🚀 View run HCPflat_large_gsrFalse__beta_sex_HCPFT at: https://stability.wandb.io/ckadirt/fMRI-foundation-model/runs/HCPflat_large_gsrFalse__beta_sex_HCPFT_83810
+wandb: Find logs at: wandb/run-20241126_213704-HCPflat_large_gsrFalse__beta_sex_HCPFT_83810/logs

541283.err

@@ -0,0 +1,54 @@
+[NbConvertApp] Converting notebook HCP_downstream_finetune.ipynb to python
+[NbConvertApp] Writing 31825 bytes to HCP_downstream_finetune.py
+/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py:658: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
+  state = torch.load(checkpoint_path)
+wandb: Using wandb-core as the SDK backend. Please refer to https://wandb.me/wandb-core for more information.
+wandb: Currently logged in as: ckadirt. Use `wandb login --relogin` to force relogin
+wandb: Tracking run with wandb version 0.18.3
+wandb: Run data is saved locally in /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_213826-HCPflat_large_gsrFalse__beta_sex_HCPFT_83810
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Resuming run HCPflat_large_gsrFalse__beta_sex_HCPFT
+wandb: ⭐️ View project at https://stability.wandb.io/ckadirt/fMRI-foundation-model
+wandb: 🚀 View run at https://stability.wandb.io/ckadirt/fMRI-foundation-model/runs/HCPflat_large_gsrFalse__beta_sex_HCPFT_83810
+
+Traceback (most recent call last):
+  File "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py", line 843, in <module>
+    outputs = model(images, gsr=gsr)  # Shape: [num_train_samples, num_classes]
+              ^^^^^^^^^^^^^^^^^^^^^^
+  File "/admin/home-ckadirt/foundation_env/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1553, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/admin/home-ckadirt/foundation_env/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1562, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py", line 696, in forward
+    x = self.mae_model(x, global_pool=global_pool, forward_features = True)
+        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/admin/home-ckadirt/foundation_env/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1553, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/admin/home-ckadirt/foundation_env/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1562, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/mae_utils/flat_models.py", line 753, in forward
+    x = blk(x)
+        ^^^^^^
+  File "/admin/home-ckadirt/foundation_env/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1553, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/admin/home-ckadirt/foundation_env/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1562, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/mae_utils/video_vit.py", line 166, in forward
+    x = x + self.drop_path(self.attn(self.norm1(x)))
+                           ^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/admin/home-ckadirt/foundation_env/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1553, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/admin/home-ckadirt/foundation_env/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1562, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/mae_utils/video_vit.py", line 114, in forward
+    attn = (q @ k.transpose(-2, -1)) * self.scale
+            ~~^~~~~~~~~~~~~~~~~~~~~
+torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 2.04 GiB. GPU 0 has a total capacity of 79.11 GiB of which 1.06 GiB is free. Including non-PyTorch memory, this process has 78.04 GiB memory in use. Of the allocated memory 75.85 GiB is allocated by PyTorch, and 1.52 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation.  See documentation for Memory Management  (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)

541283.out

@@ -0,0 +1,70 @@
+NUM_GPUS=1
+MASTER_ADDR=ip-10-0-133-32
+MASTER_PORT=17673
+WORLD_SIZE=1
+------ ARGS ------- 
+ Namespace(found_model_name='HCPflat_large_gsrFalse_', epoch_checkpoint='epoch99.pth', model_suffix='beta', hcp_flat_path='/weka/proj-medarc/shared/HCP-Flat', batch_size=24, wandb_log=True, num_epochs=20, lr_scheduler_type='cycle', save_ckpt=False, seed=42, max_lr=0.0003, target='sex', num_workers=15, weight_decay=0.001, global_pool=True)
+outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_
+Loaded config.yaml from ckpt folder /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_
+
+__CONFIG__
+base_lr = 0.001
+batch_size = 32
+ckpt_interval = 5
+ckpt_saving = True
+cls_embed = True
+contrastive_loss_weight = 1.0
+datasets_to_include = HCP
+decoder_embed_dim = 512
+grad_accumulation_steps = 1
+grad_clip = 1.0
+gsr = False
+hcp_flat_path = /weka/proj-medarc/shared/HCP-Flat
+mask_ratio = 0.75
+model_name = HCPflat_large_gsrFalse_
+no_qkv_bias = False
+norm_pix_loss = False
+nsd_flat_path = /weka/proj-medarc/shared/NSD-Flat
+num_epochs = 100
+num_frames = 16
+num_samples_per_epoch = 200000
+num_workers = 10
+patch_size = 16
+pct_masks_to_decode = 1
+plotting = True
+pred_t_dim = 8
+print_interval = 20
+probe_base_lr = 0.0003
+probe_batch_size = 8
+probe_num_epochs = 30
+probe_num_samples_per_epoch = 100000
+resume_from_ckpt = True
+seed = 42
+sep_pos_embed = True
+t_patch_size = 2
+test_num_samples_per_epoch = 50000
+test_set = False
+trunc_init = False
+use_contrastive_loss = False
+wandb_log = True
+
+
+WORLD_SIZE=1
+PID of this process = 1888215
+global_pool = True
+gsr = False
+Creating datasets
+Datasets ready
+img_size (144, 320) patch_size (16, 16) frames 16 t_patch_size 2
+model initialized
+latest_checkpoint: epoch99.pth
+
+Loaded checkpoint epoch99.pth from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_
+
+Input dimension: 1024
+total_steps 92760
+wandb_config:
+ {'model_name': 'HCPflat_large_gsrFalse__HCP_FT_sex', 'batch_size': 24, 'weight_decay': 0.001, 'num_epochs': 20, 'seed': 42, 'lr_scheduler_type': 'cycle', 'save_ckpt': False, 'max_lr': 0.0003, 'target': 'sex', 'num_workers': 15}
+wandb_id: HCPflat_large_gsrFalse__beta_sex_HCPFT_83810
+wandb: 🚀 View run HCPflat_large_gsrFalse__beta_sex_HCPFT at: https://stability.wandb.io/ckadirt/fMRI-foundation-model/runs/HCPflat_large_gsrFalse__beta_sex_HCPFT_83810
+wandb: Find logs at: wandb/run-20241126_213826-HCPflat_large_gsrFalse__beta_sex_HCPFT_83810/logs

541284.err

@@ -0,0 +1,16 @@
+[NbConvertApp] Converting notebook HCP_downstream_finetune.ipynb to python
+[NbConvertApp] Writing 31825 bytes to HCP_downstream_finetune.py
+/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py:658: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
+  state = torch.load(checkpoint_path)
+wandb: Using wandb-core as the SDK backend. Please refer to https://wandb.me/wandb-core for more information.
+wandb: Currently logged in as: ckadirt. Use `wandb login --relogin` to force relogin
+wandb: Tracking run with wandb version 0.18.3
+wandb: Run data is saved locally in /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_214117-HCPflat_large_gsrFalse__beta_sex_HCPFT_83810
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Resuming run HCPflat_large_gsrFalse__beta_sex_HCPFT
+wandb: ⭐️ View project at https://stability.wandb.io/ckadirt/fMRI-foundation-model
+wandb: 🚀 View run at https://stability.wandb.io/ckadirt/fMRI-foundation-model/runs/HCPflat_large_gsrFalse__beta_sex_HCPFT_83810
+
+slurmstepd: error: *** JOB 541284 ON ip-10-0-133-32 CANCELLED AT 2024-11-26T21:43:06 ***
+slurmstepd: error: *** REASON: burst_buffer/lua: Stage-out in progress ***
+

541284.out

@@ -0,0 +1,68 @@
+NUM_GPUS=1
+MASTER_ADDR=ip-10-0-133-32
+MASTER_PORT=16266
+WORLD_SIZE=1
+------ ARGS ------- 
+ Namespace(found_model_name='HCPflat_large_gsrFalse_', epoch_checkpoint='epoch99.pth', model_suffix='beta', hcp_flat_path='/weka/proj-medarc/shared/HCP-Flat', batch_size=16, wandb_log=True, num_epochs=20, lr_scheduler_type='cycle', save_ckpt=False, seed=42, max_lr=0.0003, target='sex', num_workers=15, weight_decay=0.001, global_pool=True)
+outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_
+Loaded config.yaml from ckpt folder /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_
+
+__CONFIG__
+base_lr = 0.001
+batch_size = 32
+ckpt_interval = 5
+ckpt_saving = True
+cls_embed = True
+contrastive_loss_weight = 1.0
+datasets_to_include = HCP
+decoder_embed_dim = 512
+grad_accumulation_steps = 1
+grad_clip = 1.0
+gsr = False
+hcp_flat_path = /weka/proj-medarc/shared/HCP-Flat
+mask_ratio = 0.75
+model_name = HCPflat_large_gsrFalse_
+no_qkv_bias = False
+norm_pix_loss = False
+nsd_flat_path = /weka/proj-medarc/shared/NSD-Flat
+num_epochs = 100
+num_frames = 16
+num_samples_per_epoch = 200000
+num_workers = 10
+patch_size = 16
+pct_masks_to_decode = 1
+plotting = True
+pred_t_dim = 8
+print_interval = 20
+probe_base_lr = 0.0003
+probe_batch_size = 8
+probe_num_epochs = 30
+probe_num_samples_per_epoch = 100000
+resume_from_ckpt = True
+seed = 42
+sep_pos_embed = True
+t_patch_size = 2
+test_num_samples_per_epoch = 50000
+test_set = False
+trunc_init = False
+use_contrastive_loss = False
+wandb_log = True
+
+
+WORLD_SIZE=1
+PID of this process = 1891122
+global_pool = True
+gsr = False
+Creating datasets
+Datasets ready
+img_size (144, 320) patch_size (16, 16) frames 16 t_patch_size 2
+model initialized
+latest_checkpoint: epoch99.pth
+
+Loaded checkpoint epoch99.pth from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_
+
+Input dimension: 1024
+total_steps 139140
+wandb_config:
+ {'model_name': 'HCPflat_large_gsrFalse__HCP_FT_sex', 'batch_size': 16, 'weight_decay': 0.001, 'num_epochs': 20, 'seed': 42, 'lr_scheduler_type': 'cycle', 'save_ckpt': False, 'max_lr': 0.0003, 'target': 'sex', 'num_workers': 15}
+wandb_id: HCPflat_large_gsrFalse__beta_sex_HCPFT_83810

541285.err

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99eca64affe9760335ce81df790b48e00296ec7821b880d9cb1e9aa3d49931f5
+size 13422040

541286.err

@@ -0,0 +1,14 @@
+[NbConvertApp] Converting notebook HCP_downstream_raw_flatmaps.ipynb to python
+[NbConvertApp] Writing 36405 bytes to HCP_downstream_raw_flatmaps.py
+usage: HCP_downstream_raw_flatmaps.py [-h] [--model_suffix MODEL_SUFFIX]
+                                      [--hcp_flat_path HCP_FLAT_PATH]
+                                      [--batch_size BATCH_SIZE]
+                                      [--wandb_log | --no-wandb_log]
+                                      [--num_epochs NUM_EPOCHS]
+                                      [--lr_scheduler_type {cycle,linear}]
+                                      [--save_ckpt | --no-save_ckpt]
+                                      [--seed SEED] [--max_lr MAX_LR]
+                                      [--target {trial_type,sex,gender}]
+                                      [--num_workers NUM_WORKERS]
+                                      [--weight_decay WEIGHT_DECAY]
+HCP_downstream_raw_flatmaps.py: error: argument --target: invalid choice: 'age' (choose from 'trial_type', 'sex', 'gender')

541286.out

@@ -0,0 +1,5 @@
+NUM_GPUS=1
+MASTER_ADDR=ip-10-0-136-5
+MASTER_PORT=18510
+WORLD_SIZE=1
+PID of this process = 1032528

541287.err

@@ -0,0 +1,21 @@
+[NbConvertApp] Converting notebook HCP_downstream_raw_flatmaps.ipynb to python
+[NbConvertApp] Writing 36402 bytes to HCP_downstream_raw_flatmaps.py
+wandb: Using wandb-core as the SDK backend. Please refer to https://wandb.me/wandb-core for more information.
+wandb: Currently logged in as: ckadirt. Use `wandb login --relogin` to force relogin
+wandb: Tracking run with wandb version 0.18.3
+wandb: Run data is saved locally in /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_215001-HCPflat_raw_beta_age_83810
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run HCPflat_raw_beta_age
+wandb: ⭐️ View project at https://stability.wandb.io/ckadirt/fMRI-foundation-model
+wandb: 🚀 View run at https://stability.wandb.io/ckadirt/fMRI-foundation-model/runs/HCPflat_raw_beta_age_83810
+
+  return F.mse_loss(input, target, reduction=self.reduction)
+
+  return F.mse_loss(input, target, reduction=self.reduction)
+
+
+  return F.mse_loss(input, target, reduction=self.reduction)
+
+
+slurmstepd: error: *** JOB 541287 ON ip-10-0-136-5 CANCELLED AT 2024-11-26T21:59:19 ***
+slurmstepd: error: *** REASON: burst_buffer/lua: Stage-out in progress ***

541287.out

@@ -0,0 +1,21 @@
+NUM_GPUS=1
+MASTER_ADDR=ip-10-0-136-5
+MASTER_PORT=17942
+WORLD_SIZE=1
+PID of this process = 1046807
+------ ARGS ------- 
+ Namespace(model_suffix='beta', hcp_flat_path='/weka/proj-medarc/shared/HCP-Flat', batch_size=128, wandb_log=True, num_epochs=20, lr_scheduler_type='cycle', save_ckpt=False, seed=42, max_lr=0.1, target='age', num_workers=15, weight_decay=1e-05)
+Input dimension: 737280
+total_steps 17400
+wandb_config:
+ {'model_name': 'HCPflat_raw_age', 'batch_size': 128, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42, 'lr_scheduler_type': 'cycle', 'save_ckpt': False, 'max_lr': 0.1, 'target': 'age', 'num_workers': 15}
+wandb_id: HCPflat_raw_beta_age_83810
+Step [100/870] - Training Loss: 924.1876 - Training MSE: 207942.1405
+Step [200/870] - Training Loss: 2762.3567 - Training MSE: 206121.1501
+Step [300/870] - Training Loss: 22823.8477 - Training MSE: 690589.0472
+Step [400/870] - Training Loss: 99405.8828 - Training MSE: 1767819.3204
+Step [500/870] - Training Loss: 214733.7188 - Training MSE: 4693776.8283
+Step [600/870] - Training Loss: 263763.7500 - Training MSE: 8583782.2903
+Step [700/870] - Training Loss: 397232.1875 - Training MSE: 13775790.2974
+Step [800/870] - Training Loss: 665133.5000 - Training MSE: 21325931.7302
+Epoch [1/20] - Training Loss: 229616.0744, Training MSE: 29363571.6190 - Validation Loss: 1573111.9296, Validation MSE: 200990315.8497

541290.err

@@ -0,0 +1,18 @@
+[NbConvertApp] Converting notebook HCP_downstream_raw_flatmaps.ipynb to python
+[NbConvertApp] Writing 36445 bytes to HCP_downstream_raw_flatmaps.py
+wandb: Using wandb-core as the SDK backend. Please refer to https://wandb.me/wandb-core for more information.
+wandb: Currently logged in as: ckadirt. Use `wandb login --relogin` to force relogin
+wandb: Tracking run with wandb version 0.18.3
+wandb: Run data is saved locally in /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241126_220105-HCPflat_raw_beta_age_7cc4d250-ac94-488c-bec4-39b422ee70de
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run HCPflat_raw_beta_age
+wandb: ⭐️ View project at https://stability.wandb.io/ckadirt/fMRI-foundation-model
+wandb: 🚀 View run at https://stability.wandb.io/ckadirt/fMRI-foundation-model/runs/HCPflat_raw_beta_age_7cc4d250-ac94-488c-bec4-39b422ee70de
+
+  return F.mse_loss(input, target, reduction=self.reduction)
+
+  return F.mse_loss(input, target, reduction=self.reduction)
+
+
+slurmstepd: error: *** JOB 541290 ON ip-10-0-136-5 CANCELLED AT 2024-11-26T22:08:55 ***
+slurmstepd: error: *** REASON: burst_buffer/lua: Stage-out in progress ***

541290.out

@@ -0,0 +1,20 @@
+NUM_GPUS=1
+MASTER_ADDR=ip-10-0-136-5
+MASTER_PORT=11785
+WORLD_SIZE=1
+PID of this process = 1101409
+------ ARGS ------- 
+ Namespace(model_suffix='beta', hcp_flat_path='/weka/proj-medarc/shared/HCP-Flat', batch_size=128, wandb_log=True, num_epochs=20, lr_scheduler_type='cycle', save_ckpt=False, seed=42, max_lr=0.001, target='age', num_workers=15, weight_decay=1e-05)
+Input dimension: 737280
+total_steps 17400
+wandb_config:
+ {'model_name': 'HCPflat_raw_age', 'batch_size': 128, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42, 'lr_scheduler_type': 'cycle', 'save_ckpt': False, 'max_lr': 0.001, 'target': 'age', 'num_workers': 15}
+wandb_id: HCPflat_raw_beta_age_7cc4d250-ac94-488c-bec4-39b422ee70de
+Step [100/870] - Training Loss: 0.7004 - Training MSE: 98.7605
+Step [200/870] - Training Loss: 0.9553 - Training MSE: 104.1519
+Step [300/870] - Training Loss: 1.9018 - Training MSE: 129.4920
+Step [400/870] - Training Loss: 5.3471 - Training MSE: 222.5458
+Step [500/870] - Training Loss: 15.9923 - Training MSE: 460.4371
+Step [600/870] - Training Loss: 32.0503 - Training MSE: 927.5124
+Step [700/870] - Training Loss: 70.9650 - Training MSE: 1574.1360
+Step [800/870] - Training Loss: 80.0552 - Training MSE: 2592.0538

541293.out

@@ -0,0 +1,194 @@
+NUM_GPUS=1
+MASTER_ADDR=ip-10-0-131-135
+MASTER_PORT=18962
+WORLD_SIZE=1
+PID of this process = 1324243
+------ ARGS ------- 
+ Namespace(model_suffix='beta', hcp_flat_path='/weka/proj-medarc/shared/HCP-Flat', batch_size=128, wandb_log=True, num_epochs=20, lr_scheduler_type='cycle', save_ckpt=False, seed=42, max_lr=0.001, target='age', num_workers=15, weight_decay=1e-05)
+Input dimension: 737280
+total_steps 17400
+wandb_config:
+ {'model_name': 'HCPflat_raw_age', 'batch_size': 128, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42, 'lr_scheduler_type': 'cycle', 'save_ckpt': False, 'max_lr': 0.001, 'target': 'age', 'num_workers': 15}
+wandb_id: HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271
+Step [100/870] - Training Loss: 0.7004 - Training MSE: 98.7605
+Step [200/870] - Training Loss: 0.9553 - Training MSE: 104.1519
+Step [300/870] - Training Loss: 1.9018 - Training MSE: 129.4920
+Step [400/870] - Training Loss: 5.3471 - Training MSE: 222.5458
+Step [500/870] - Training Loss: 15.9923 - Training MSE: 460.4371
+Step [600/870] - Training Loss: 32.0503 - Training MSE: 927.5124
+Step [700/870] - Training Loss: 70.9650 - Training MSE: 1574.1360
+Step [800/870] - Training Loss: 80.0552 - Training MSE: 2592.0538
+Epoch [1/20] - Training Loss: 27.4343, Training MSE: 3507.0528 - Validation Loss: 131.3325, Validation MSE: 16734.0162
+Step [100/870] - Training Loss: 225.2205 - Training MSE: 30714.7531
+Step [200/870] - Training Loss: 292.3234 - Training MSE: 33573.5262
+Step [300/870] - Training Loss: 292.7283 - Training MSE: 36038.9369
+Step [400/870] - Training Loss: 299.2356 - Training MSE: 37454.6156
+Step [500/870] - Training Loss: 443.3794 - Training MSE: 38764.3162
+Step [600/870] - Training Loss: 427.7180 - Training MSE: 41074.9210
+Step [700/870] - Training Loss: 394.7606 - Training MSE: 42823.9646
+Step [800/870] - Training Loss: 427.3995 - Training MSE: 44576.2413
+Epoch [2/20] - Training Loss: 355.6319, Training MSE: 45505.3376 - Validation Loss: 424.2622, Validation MSE: 54209.6258
+Step [100/870] - Training Loss: 616.6658 - Training MSE: 91859.6765
+Step [200/870] - Training Loss: 594.7969 - Training MSE: 88332.5994
+Step [300/870] - Training Loss: 599.6354 - Training MSE: 89295.5247
+Step [400/870] - Training Loss: 575.7101 - Training MSE: 88032.4892
+Step [500/870] - Training Loss: 503.4290 - Training MSE: 87341.0175
+Step [600/870] - Training Loss: 449.5532 - Training MSE: 86012.6966
+Step [700/870] - Training Loss: 693.0297 - Training MSE: 85016.7206
+Step [800/870] - Training Loss: 551.1360 - Training MSE: 83246.9525
+Epoch [3/20] - Training Loss: 641.0600, Training MSE: 82029.1618 - Validation Loss: 380.8437, Validation MSE: 48652.5277
+Step [100/870] - Training Loss: 595.5380 - Training MSE: 66653.8484
+Step [200/870] - Training Loss: 620.8188 - Training MSE: 64999.0662
+Step [300/870] - Training Loss: 814.8269 - Training MSE: 64970.9618
+Step [400/870] - Training Loss: 465.9383 - Training MSE: 65667.7809
+Step [500/870] - Training Loss: 468.5602 - Training MSE: 64540.3150
+Step [600/870] - Training Loss: 567.9731 - Training MSE: 63776.4097
+Step [700/870] - Training Loss: 458.1937 - Training MSE: 62471.3650
+Step [800/870] - Training Loss: 486.8877 - Training MSE: 61673.2599
+Epoch [4/20] - Training Loss: 475.5952, Training MSE: 60863.9278 - Validation Loss: 230.0277, Validation MSE: 29384.7977
+Step [100/870] - Training Loss: 312.2596 - Training MSE: 43578.8527
+Step [200/870] - Training Loss: 289.8408 - Training MSE: 43494.8653
+Step [300/870] - Training Loss: 367.2291 - Training MSE: 42418.5304
+Step [400/870] - Training Loss: 367.6751 - Training MSE: 42794.9546
+Step [500/870] - Training Loss: 250.6227 - Training MSE: 42133.1672
+Step [600/870] - Training Loss: 224.6889 - Training MSE: 41366.5701
+Step [700/870] - Training Loss: 297.7198 - Training MSE: 40742.5247
+Step [800/870] - Training Loss: 285.0359 - Training MSE: 40988.5822
+Epoch [5/20] - Training Loss: 320.3213, Training MSE: 40992.2368 - Validation Loss: 343.8250, Validation MSE: 43935.6901
+Step [100/870] - Training Loss: 403.4899 - Training MSE: 41564.5185
+Step [200/870] - Training Loss: 280.5116 - Training MSE: 38702.8446
+Step [300/870] - Training Loss: 197.1333 - Training MSE: 37270.6040
+Step [400/870] - Training Loss: 219.3125 - Training MSE: 36088.7097
+Step [500/870] - Training Loss: 214.0476 - Training MSE: 36101.8102
+Step [600/870] - Training Loss: 249.8342 - Training MSE: 35435.7052
+Step [700/870] - Training Loss: 253.3519 - Training MSE: 35137.2864
+Step [800/870] - Training Loss: 260.9366 - Training MSE: 35056.1371
+Epoch [6/20] - Training Loss: 272.7941, Training MSE: 34906.8847 - Validation Loss: 181.5033, Validation MSE: 23178.8234
+Step [100/870] - Training Loss: 259.5545 - Training MSE: 41335.3267
+Step [200/870] - Training Loss: 267.8753 - Training MSE: 38545.9863
+Step [300/870] - Training Loss: 279.0856 - Training MSE: 38170.9948
+Step [400/870] - Training Loss: 253.6445 - Training MSE: 36542.9225
+Step [500/870] - Training Loss: 270.2076 - Training MSE: 35589.3113
+Step [600/870] - Training Loss: 238.1767 - Training MSE: 34781.3647
+Step [700/870] - Training Loss: 210.1095 - Training MSE: 34110.7147
+Step [800/870] - Training Loss: 170.0861 - Training MSE: 33597.5924
+Epoch [7/20] - Training Loss: 260.1132, Training MSE: 33286.1531 - Validation Loss: 149.1776, Validation MSE: 19066.1705
+Step [100/870] - Training Loss: 213.6669 - Training MSE: 30238.3354
+Step [200/870] - Training Loss: 280.7238 - Training MSE: 37416.2637
+Step [300/870] - Training Loss: 199.6157 - Training MSE: 35362.4516
+Step [400/870] - Training Loss: 293.6560 - Training MSE: 34585.9442
+Step [500/870] - Training Loss: 243.5896 - Training MSE: 33888.9126
+Step [600/870] - Training Loss: 250.7629 - Training MSE: 33711.0958
+Step [700/870] - Training Loss: 204.3937 - Training MSE: 33619.7849
+Step [800/870] - Training Loss: 297.2823 - Training MSE: 32938.8256
+Epoch [8/20] - Training Loss: 253.8449, Training MSE: 32483.8266 - Validation Loss: 138.4049, Validation MSE: 17682.2886
+Step [100/870] - Training Loss: 185.7414 - Training MSE: 26811.0968
+Step [200/870] - Training Loss: 218.8939 - Training MSE: 26100.9550
+Step [300/870] - Training Loss: 162.8873 - Training MSE: 26417.2719
+Step [400/870] - Training Loss: 161.6969 - Training MSE: 25955.3044
+Step [500/870] - Training Loss: 207.4261 - Training MSE: 25493.3734
+Step [600/870] - Training Loss: 187.3801 - Training MSE: 25508.5498
+Step [700/870] - Training Loss: 214.4618 - Training MSE: 25381.5527
+Step [800/870] - Training Loss: 180.7133 - Training MSE: 25397.3536
+Epoch [9/20] - Training Loss: 196.7826, Training MSE: 25180.1054 - Validation Loss: 116.9404, Validation MSE: 14922.8448
+Step [100/870] - Training Loss: 156.8096 - Training MSE: 21927.3050
+Step [200/870] - Training Loss: 150.6434 - Training MSE: 20166.1289
+Step [300/870] - Training Loss: 156.5338 - Training MSE: 19752.6387
+Step [400/870] - Training Loss: 115.4449 - Training MSE: 19134.1077
+Step [500/870] - Training Loss: 151.5466 - Training MSE: 18829.4417
+Step [600/870] - Training Loss: 133.8123 - Training MSE: 19151.8094
+Step [700/870] - Training Loss: 150.4475 - Training MSE: 19053.8740
+Step [800/870] - Training Loss: 134.3665 - Training MSE: 18898.9213
+Epoch [10/20] - Training Loss: 145.9114, Training MSE: 18671.6165 - Validation Loss: 84.6188, Validation MSE: 10794.5938
+Step [100/870] - Training Loss: 110.3017 - Training MSE: 14012.7743
+Step [200/870] - Training Loss: 89.6949 - Training MSE: 13419.7123
+Step [300/870] - Training Loss: 99.2646 - Training MSE: 13079.7575
+Step [400/870] - Training Loss: 93.9918 - Training MSE: 12765.8371
+Step [500/870] - Training Loss: 88.8151 - Training MSE: 12486.5892
+Step [600/870] - Training Loss: 143.9832 - Training MSE: 12778.8930
+Step [700/870] - Training Loss: 98.1826 - Training MSE: 12722.4271
+Step [800/870] - Training Loss: 69.2268 - Training MSE: 12515.3702
+Epoch [11/20] - Training Loss: 97.2335, Training MSE: 12443.2557 - Validation Loss: 49.3342, Validation MSE: 6300.2935
+Step [100/870] - Training Loss: 84.1289 - Training MSE: 12731.3686
+Step [200/870] - Training Loss: 101.6131 - Training MSE: 11380.9970
+Step [300/870] - Training Loss: 68.8949 - Training MSE: 10543.4813
+Step [400/870] - Training Loss: 67.8553 - Training MSE: 9954.5183
+Step [500/870] - Training Loss: 69.3201 - Training MSE: 9525.7463
+Step [600/870] - Training Loss: 61.4179 - Training MSE: 9232.0457
+Step [700/870] - Training Loss: 58.9125 - Training MSE: 8939.9569
+Step [800/870] - Training Loss: 61.9412 - Training MSE: 8919.2549
+Epoch [12/20] - Training Loss: 68.8162, Training MSE: 8806.7756 - Validation Loss: 34.1533, Validation MSE: 4359.8410
+Step [100/870] - Training Loss: 39.4785 - Training MSE: 5607.7454
+Step [200/870] - Training Loss: 43.6887 - Training MSE: 5866.7379
+Step [300/870] - Training Loss: 51.5316 - Training MSE: 5913.9918
+Step [400/870] - Training Loss: 44.5591 - Training MSE: 5827.7686
+Step [500/870] - Training Loss: 56.4356 - Training MSE: 5818.0397
+Step [600/870] - Training Loss: 64.3104 - Training MSE: 6190.6625
+Step [700/870] - Training Loss: 45.7098 - Training MSE: 6203.7776
+Step [800/870] - Training Loss: 37.0419 - Training MSE: 6119.6922
+Epoch [13/20] - Training Loss: 47.2533, Training MSE: 6047.1326 - Validation Loss: 24.4559, Validation MSE: 3123.7767
+Step [100/870] - Training Loss: 24.0725 - Training MSE: 4543.2675
+Step [200/870] - Training Loss: 33.1887 - Training MSE: 4195.8679
+Step [300/870] - Training Loss: 27.0619 - Training MSE: 4041.4841
+Step [400/870] - Training Loss: 39.4217 - Training MSE: 4002.0279
+Step [500/870] - Training Loss: 31.7641 - Training MSE: 4235.0661
+Step [600/870] - Training Loss: 25.3477 - Training MSE: 4154.7053
+Step [700/870] - Training Loss: 28.7334 - Training MSE: 4054.2543
+Step [800/870] - Training Loss: 25.1178 - Training MSE: 3951.9298
+Epoch [14/20] - Training Loss: 30.4371, Training MSE: 3894.8694 - Validation Loss: 15.2119, Validation MSE: 1942.3499
+Step [100/870] - Training Loss: 19.7150 - Training MSE: 2138.6110
+Step [200/870] - Training Loss: 14.8900 - Training MSE: 2099.8135
+Step [300/870] - Training Loss: 14.9437 - Training MSE: 2029.9849
+Step [400/870] - Training Loss: 16.5218 - Training MSE: 1988.6470
+Step [500/870] - Training Loss: 13.5388 - Training MSE: 1959.2617
+Step [600/870] - Training Loss: 12.7145 - Training MSE: 1932.0582
+Step [700/870] - Training Loss: 15.4000 - Training MSE: 1935.8872
+Step [800/870] - Training Loss: 12.4422 - Training MSE: 2013.0126
+Epoch [15/20] - Training Loss: 15.5981, Training MSE: 1996.1209 - Validation Loss: 7.5625, Validation MSE: 966.0334
+Step [100/870] - Training Loss: 10.2901 - Training MSE: 2687.8184
+Step [200/870] - Training Loss: 11.1747 - Training MSE: 1969.2730
+Step [300/870] - Training Loss: 6.5812 - Training MSE: 1654.0797
+Step [400/870] - Training Loss: 7.1049 - Training MSE: 1479.5801
+Step [500/870] - Training Loss: 6.1494 - Training MSE: 1376.3151
+Step [600/870] - Training Loss: 8.5310 - Training MSE: 1297.4019
+Step [700/870] - Training Loss: 7.0269 - Training MSE: 1241.7873
+Step [800/870] - Training Loss: 7.1375 - Training MSE: 1190.3315
+Epoch [16/20] - Training Loss: 9.0716, Training MSE: 1160.8802 - Validation Loss: 3.7649, Validation MSE: 481.1624
+Step [100/870] - Training Loss: 3.1061 - Training MSE: 448.2485
+Step [200/870] - Training Loss: 2.6831 - Training MSE: 443.6960
+Step [300/870] - Training Loss: 3.3528 - Training MSE: 438.0512
+Step [400/870] - Training Loss: 9.4740 - Training MSE: 675.8739
+Step [500/870] - Training Loss: 4.1480 - Training MSE: 691.1889
+Step [600/870] - Training Loss: 4.2169 - Training MSE: 668.9939
+Step [700/870] - Training Loss: 3.0901 - Training MSE: 646.7708
+Step [800/870] - Training Loss: 5.0069 - Training MSE: 628.2751
+Epoch [17/20] - Training Loss: 4.8296, Training MSE: 617.9883 - Validation Loss: 2.7624, Validation MSE: 352.8308
+Step [100/870] - Training Loss: 2.7670 - Training MSE: 320.1880
+Step [200/870] - Training Loss: 1.6207 - Training MSE: 287.7513
+Step [300/870] - Training Loss: 1.9942 - Training MSE: 275.3315
+Step [400/870] - Training Loss: 2.2564 - Training MSE: 270.3133
+Step [500/870] - Training Loss: 1.6254 - Training MSE: 266.9202
+Step [600/870] - Training Loss: 1.7127 - Training MSE: 264.0805
+Step [700/870] - Training Loss: 1.5853 - Training MSE: 261.9605
+Step [800/870] - Training Loss: 1.9506 - Training MSE: 261.0231
+Epoch [18/20] - Training Loss: 2.0329, Training MSE: 260.1193 - Validation Loss: 1.7839, Validation MSE: 227.7603
+Step [100/870] - Training Loss: 1.1409 - Training MSE: 143.1174
+Step [200/870] - Training Loss: 1.1094 - Training MSE: 152.5059
+Step [300/870] - Training Loss: 1.2674 - Training MSE: 157.0374
+Step [400/870] - Training Loss: 1.0359 - Training MSE: 155.5929
+Step [500/870] - Training Loss: 1.1091 - Training MSE: 155.0004
+Step [600/870] - Training Loss: 1.1539 - Training MSE: 155.3943
+Step [700/870] - Training Loss: 1.2641 - Training MSE: 155.1105
+Step [800/870] - Training Loss: 1.2053 - Training MSE: 155.1447
+Epoch [19/20] - Training Loss: 1.2139, Training MSE: 155.3429 - Validation Loss: 1.5769, Validation MSE: 201.3158
+Step [100/870] - Training Loss: 1.0962 - Training MSE: 117.9358
+Step [200/870] - Training Loss: 0.8717 - Training MSE: 117.8234
+Step [300/870] - Training Loss: 0.8542 - Training MSE: 118.7729
+Step [400/870] - Training Loss: 1.0950 - Training MSE: 119.2174
+Step [500/870] - Training Loss: 1.1080 - Training MSE: 119.1448
+Step [600/870] - Training Loss: 1.0546 - Training MSE: 118.9895
+Step [700/870] - Training Loss: 0.8502 - Training MSE: 119.4526
+Step [800/870] - Training Loss: 0.7913 - Training MSE: 119.7514
+Epoch [20/20] - Training Loss: 0.9381, Training MSE: 120.0448 - Validation Loss: 1.5444, Validation MSE: 197.2017
+wandb: 🚀 View run HCPflat_raw_beta_age at: https://stability.wandb.io/ckadirt/fMRI-foundation-model/runs/HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271
+wandb: Find logs at: wandb/run-20241126_221003-HCPflat_raw_beta_age_9a3e14f1-ec90-47c9-a06e-a395872f2271/logs

541294.out

@@ -0,0 +1,208 @@
+NUM_GPUS=1
+MASTER_ADDR=ip-10-0-135-126
+MASTER_PORT=16509
+WORLD_SIZE=1
+------ ARGS ------- 
+ Namespace(found_model_name='HCPflat_large_gsrFalse_', epoch_checkpoint='epoch99.pth', model_suffix='beta', hcp_flat_path='/weka/proj-medarc/shared/HCP-Flat', batch_size=16, wandb_log=True, num_epochs=20, lr_scheduler_type='cycle', save_ckpt=False, seed=42, max_lr=3e-05, target='age', num_workers=15, weight_decay=0.001, global_pool=True)
+outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_
+Loaded config.yaml from ckpt folder /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_
+
+__CONFIG__
+base_lr = 0.001
+batch_size = 32
+ckpt_interval = 5
+ckpt_saving = True
+cls_embed = True
+contrastive_loss_weight = 1.0
+datasets_to_include = HCP
+decoder_embed_dim = 512
+grad_accumulation_steps = 1
+grad_clip = 1.0
+gsr = False
+hcp_flat_path = /weka/proj-medarc/shared/HCP-Flat
+mask_ratio = 0.75
+model_name = HCPflat_large_gsrFalse_
+no_qkv_bias = False
+norm_pix_loss = False
+nsd_flat_path = /weka/proj-medarc/shared/NSD-Flat
+num_epochs = 100
+num_frames = 16
+num_samples_per_epoch = 200000
+num_workers = 10
+patch_size = 16
+pct_masks_to_decode = 1
+plotting = True
+pred_t_dim = 8
+print_interval = 20
+probe_base_lr = 0.0003
+probe_batch_size = 8
+probe_num_epochs = 30
+probe_num_samples_per_epoch = 100000
+resume_from_ckpt = True
+seed = 42
+sep_pos_embed = True
+t_patch_size = 2
+test_num_samples_per_epoch = 50000
+test_set = False
+trunc_init = False
+use_contrastive_loss = False
+wandb_log = True
+
+
+WORLD_SIZE=1
+PID of this process = 2074741
+global_pool = True
+gsr = False
+Creating datasets
+Datasets ready
+img_size (144, 320) patch_size (16, 16) frames 16 t_patch_size 2
+model initialized
+latest_checkpoint: epoch99.pth
+
+Loaded checkpoint epoch99.pth from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_
+
+Input dimension: 1024
+total_steps 139140
+wandb_config:
+ {'model_name': 'HCPflat_large_gsrFalse__HCP_FT_age', 'batch_size': 16, 'weight_decay': 0.001, 'num_epochs': 20, 'seed': 42, 'lr_scheduler_type': 'cycle', 'save_ckpt': False, 'max_lr': 3e-05, 'target': 'age', 'num_workers': 15}
+wandb_id: HCPflat_large_gsrFalse__beta_age_HCPFT_185e68b7-ea11-4f13-b6c7-a9ecc17084b1
+Step [100/6957] - Training Loss: 0.3892 - Training MSE: 8.0130
+Step [200/6957] - Training Loss: 0.4409 - Training MSE: 7.4482
+Step [300/6957] - Training Loss: 0.5592 - Training MSE: 7.3383
+Step [400/6957] - Training Loss: 0.5652 - Training MSE: 7.1366
+Step [500/6957] - Training Loss: 0.3701 - Training MSE: 7.0621
+Step [600/6957] - Training Loss: 0.3159 - Training MSE: 6.9860
+Step [700/6957] - Training Loss: 0.3431 - Training MSE: 6.9015
+Step [800/6957] - Training Loss: 0.5229 - Training MSE: 6.8741
+Step [900/6957] - Training Loss: 0.6388 - Training MSE: 6.8505
+Step [1000/6957] - Training Loss: 0.5065 - Training MSE: 6.8173
+Step [1100/6957] - Training Loss: 0.4181 - Training MSE: 6.8281
+Step [1200/6957] - Training Loss: 0.3158 - Training MSE: 6.8081
+Step [1300/6957] - Training Loss: 0.3294 - Training MSE: 6.8239
+Step [1400/6957] - Training Loss: 0.4973 - Training MSE: 6.8025
+Step [1500/6957] - Training Loss: 0.3820 - Training MSE: 6.7897
+Step [1600/6957] - Training Loss: 0.3106 - Training MSE: 6.7547
+Step [1700/6957] - Training Loss: 0.4086 - Training MSE: 6.7618
+Step [1800/6957] - Training Loss: 0.5164 - Training MSE: 6.7494
+Step [1900/6957] - Training Loss: 0.3825 - Training MSE: 6.7591
+Step [2000/6957] - Training Loss: 0.7149 - Training MSE: 6.7480
+Step [2100/6957] - Training Loss: 0.3802 - Training MSE: 6.7397
+Step [2200/6957] - Training Loss: 0.3562 - Training MSE: 6.7415
+Step [2300/6957] - Training Loss: 0.2990 - Training MSE: 6.7273
+Step [2400/6957] - Training Loss: 0.6899 - Training MSE: 6.7225
+Step [2500/6957] - Training Loss: 0.4890 - Training MSE: 6.7216
+Step [2600/6957] - Training Loss: 0.3221 - Training MSE: 6.7234
+Step [2700/6957] - Training Loss: 0.4387 - Training MSE: 6.7172
+Step [2800/6957] - Training Loss: 0.5289 - Training MSE: 6.7050
+Step [2900/6957] - Training Loss: 0.3853 - Training MSE: 6.6949
+Step [3000/6957] - Training Loss: 0.5635 - Training MSE: 6.6938
+Step [3100/6957] - Training Loss: 0.2955 - Training MSE: 6.6832
+Step [3200/6957] - Training Loss: 0.4344 - Training MSE: 6.6789
+Step [3300/6957] - Training Loss: 0.6001 - Training MSE: 6.6725
+Step [3400/6957] - Training Loss: 0.5012 - Training MSE: 6.6781
+Step [3500/6957] - Training Loss: 0.4927 - Training MSE: 6.6781
+Step [3600/6957] - Training Loss: 0.4414 - Training MSE: 6.6745
+Step [3700/6957] - Training Loss: 0.5066 - Training MSE: 6.6728
+Step [3800/6957] - Training Loss: 0.2950 - Training MSE: 6.6748
+Step [3900/6957] - Training Loss: 0.1609 - Training MSE: 6.6664
+Step [4000/6957] - Training Loss: 0.5618 - Training MSE: 6.6716
+Step [4100/6957] - Training Loss: 0.3337 - Training MSE: 6.6691
+Step [4200/6957] - Training Loss: 0.2716 - Training MSE: 6.6694
+Step [4300/6957] - Training Loss: 0.5301 - Training MSE: 6.6661
+Step [4400/6957] - Training Loss: 0.4866 - Training MSE: 6.6675
+Step [4500/6957] - Training Loss: 0.5047 - Training MSE: 6.6645
+Step [4600/6957] - Training Loss: 0.6342 - Training MSE: 6.6686
+Step [4700/6957] - Training Loss: 0.2574 - Training MSE: 6.6664
+Step [4800/6957] - Training Loss: 0.3241 - Training MSE: 6.6717
+Step [4900/6957] - Training Loss: 0.3252 - Training MSE: 6.6633
+Step [5000/6957] - Training Loss: 0.3624 - Training MSE: 6.6600
+Step [5100/6957] - Training Loss: 0.1873 - Training MSE: 6.6616
+Step [5200/6957] - Training Loss: 0.2979 - Training MSE: 6.6636
+Step [5300/6957] - Training Loss: 0.7098 - Training MSE: 6.6670
+Step [5400/6957] - Training Loss: 0.4409 - Training MSE: 6.6670
+Step [5500/6957] - Training Loss: 0.7333 - Training MSE: 6.6724
+Step [5600/6957] - Training Loss: 0.4391 - Training MSE: 6.6729
+Step [5700/6957] - Training Loss: 0.2962 - Training MSE: 6.6756
+Step [5800/6957] - Training Loss: 0.2363 - Training MSE: 6.6706
+Step [5900/6957] - Training Loss: 0.2784 - Training MSE: 6.6740
+Step [6000/6957] - Training Loss: 0.2351 - Training MSE: 6.6702
+Step [6100/6957] - Training Loss: 0.3995 - Training MSE: 6.6634
+Step [6200/6957] - Training Loss: 0.3206 - Training MSE: 6.6642
+Step [6300/6957] - Training Loss: 0.6845 - Training MSE: 6.6648
+Step [6400/6957] - Training Loss: 0.2888 - Training MSE: 6.6643
+Step [6500/6957] - Training Loss: 0.3775 - Training MSE: 6.6611
+Step [6600/6957] - Training Loss: 0.4634 - Training MSE: 6.6647
+Step [6700/6957] - Training Loss: 0.3760 - Training MSE: 6.6631
+Step [6800/6957] - Training Loss: 0.4631 - Training MSE: 6.6617
+Step [6900/6957] - Training Loss: 0.5500 - Training MSE: 6.6584
+Epoch [1/20] - Training Loss: 0.4160, Training MSE: 6.6562 - Validation Loss: 0.3768, Validation MSE: 6.0284
+Step [100/6957] - Training Loss: 0.3951 - Training MSE: 6.8452
+Step [200/6957] - Training Loss: 0.3693 - Training MSE: 6.8970
+Step [300/6957] - Training Loss: 0.2880 - Training MSE: 6.7960
+Step [400/6957] - Training Loss: 0.3614 - Training MSE: 6.6778
+Step [500/6957] - Training Loss: 0.3836 - Training MSE: 6.7002
+Step [600/6957] - Training Loss: 0.2527 - Training MSE: 6.6670
+Step [700/6957] - Training Loss: 0.5024 - Training MSE: 6.6715
+Step [800/6957] - Training Loss: 0.2082 - Training MSE: 6.6020
+Step [900/6957] - Training Loss: 0.3287 - Training MSE: 6.6174
+Step [1000/6957] - Training Loss: 0.3772 - Training MSE: 6.6493
+Step [1100/6957] - Training Loss: 0.3960 - Training MSE: 6.6507
+Step [1200/6957] - Training Loss: 0.3269 - Training MSE: 6.6264
+Step [1300/6957] - Training Loss: 0.2354 - Training MSE: 6.6205
+Step [1400/6957] - Training Loss: 0.7361 - Training MSE: 6.6152
+Step [1500/6957] - Training Loss: 0.4436 - Training MSE: 6.6176
+Step [1600/6957] - Training Loss: 0.3962 - Training MSE: 6.6356
+Step [1700/6957] - Training Loss: 0.3149 - Training MSE: 6.6642
+Step [1800/6957] - Training Loss: 0.3905 - Training MSE: 6.6755
+Step [1900/6957] - Training Loss: 0.4452 - Training MSE: 6.6547
+Step [2000/6957] - Training Loss: 0.3680 - Training MSE: 6.6425
+Step [2100/6957] - Training Loss: 0.1843 - Training MSE: 6.6410
+Step [2200/6957] - Training Loss: 0.5044 - Training MSE: 6.6335
+Step [2300/6957] - Training Loss: 0.3778 - Training MSE: 6.6386
+Step [2400/6957] - Training Loss: 0.5021 - Training MSE: 6.6491
+Step [2500/6957] - Training Loss: 0.4191 - Training MSE: 6.6336
+Step [2600/6957] - Training Loss: 0.2548 - Training MSE: 6.6459
+Step [2700/6957] - Training Loss: 0.2546 - Training MSE: 6.6442
+Step [2800/6957] - Training Loss: 0.1904 - Training MSE: 6.6486
+Step [2900/6957] - Training Loss: 0.2516 - Training MSE: 6.6311
+Step [3000/6957] - Training Loss: 0.4126 - Training MSE: 6.6303
+Step [3100/6957] - Training Loss: 0.4081 - Training MSE: 6.6321
+Step [3200/6957] - Training Loss: 0.5542 - Training MSE: 6.6396
+Step [3300/6957] - Training Loss: 0.3723 - Training MSE: 6.6398
+Step [3400/6957] - Training Loss: 0.5006 - Training MSE: 6.6385
+Step [3500/6957] - Training Loss: 0.3165 - Training MSE: 6.6386
+Step [3600/6957] - Training Loss: 0.3776 - Training MSE: 6.6369
+Step [3700/6957] - Training Loss: 0.1849 - Training MSE: 6.6355
+Step [3800/6957] - Training Loss: 0.4176 - Training MSE: 6.6365
+Step [3900/6957] - Training Loss: 0.3486 - Training MSE: 6.6399
+Step [4000/6957] - Training Loss: 0.2917 - Training MSE: 6.6375
+Step [4100/6957] - Training Loss: 0.2614 - Training MSE: 6.6338
+Step [4200/6957] - Training Loss: 0.7503 - Training MSE: 6.6331
+Step [4300/6957] - Training Loss: 0.5870 - Training MSE: 6.6363
+Step [4400/6957] - Training Loss: 0.5433 - Training MSE: 6.6321
+Step [4500/6957] - Training Loss: 0.4100 - Training MSE: 6.6228
+Step [4600/6957] - Training Loss: 0.3226 - Training MSE: 6.6281
+Step [4700/6957] - Training Loss: 0.3344 - Training MSE: 6.6312
+Step [4800/6957] - Training Loss: 0.4366 - Training MSE: 6.6283
+Step [4900/6957] - Training Loss: 0.4385 - Training MSE: 6.6264
+Step [5000/6957] - Training Loss: 0.3093 - Training MSE: 6.6253
+Step [5100/6957] - Training Loss: 0.3095 - Training MSE: 6.6257
+Step [5200/6957] - Training Loss: 0.3786 - Training MSE: 6.6220
+Step [5300/6957] - Training Loss: 0.5198 - Training MSE: 6.6179
+Step [5400/6957] - Training Loss: 0.3992 - Training MSE: 6.6198
+Step [5500/6957] - Training Loss: 0.1849 - Training MSE: 6.6086
+Step [5600/6957] - Training Loss: 0.3177 - Training MSE: 6.6133
+Step [5700/6957] - Training Loss: 0.3840 - Training MSE: 6.6121
+Step [5800/6957] - Training Loss: 0.4490 - Training MSE: 6.6044
+Step [5900/6957] - Training Loss: 0.5593 - Training MSE: 6.6077
+Step [6000/6957] - Training Loss: 0.2385 - Training MSE: 6.6079
+Step [6100/6957] - Training Loss: 0.2349 - Training MSE: 6.5971
+Step [6200/6957] - Training Loss: 0.1838 - Training MSE: 6.5991
+Step [6300/6957] - Training Loss: 0.5872 - Training MSE: 6.6040
+Step [6400/6957] - Training Loss: 0.4445 - Training MSE: 6.5962
+Step [6500/6957] - Training Loss: 0.3096 - Training MSE: 6.5968
+Step [6600/6957] - Training Loss: 0.2932 - Training MSE: 6.5956
+Step [6700/6957] - Training Loss: 0.3872 - Training MSE: 6.5955
+Step [6800/6957] - Training Loss: 0.5766 - Training MSE: 6.5942
+Step [6900/6957] - Training Loss: 0.2552 - Training MSE: 6.5962
+Epoch [2/20] - Training Loss: 0.4122, Training MSE: 6.5952 - Validation Loss: 0.4057, Validation MSE: 6.4909

541338.err

@@ -0,0 +1,5 @@
+[NbConvertApp] Converting notebook HCP_downstream_raw_flatmaps.ipynb to python
+[NbConvertApp] Writing 36445 bytes to HCP_downstream_raw_flatmaps.py
+slurmstepd: error: *** REASON: burst_buffer/lua: Stage-out in progress ***
+slurmstepd: error: *** JOB 541338 ON ip-10-0-131-135 CANCELLED AT 2024-11-27T01:30:50 ***
+slurmstepd: error: *** REASON: burst_buffer/lua: Stage-out in progress ***

541338.out

@@ -0,0 +1,4 @@
+NUM_GPUS=1
+MASTER_ADDR=ip-10-0-131-135
+MASTER_PORT=15954
+WORLD_SIZE=1

541339.out

@@ -0,0 +1,46 @@
+NUM_GPUS=1
+MASTER_ADDR=ip-10-0-131-135
+MASTER_PORT=15796
+WORLD_SIZE=1
+PID of this process = 1395696
+------ ARGS ------- 
+ Namespace(model_suffix='beta', hcp_flat_path='/weka/proj-medarc/shared/HCP-Flat', batch_size=256, wandb_log=True, num_epochs=50, lr_scheduler_type='cycle', save_ckpt=False, seed=42, max_lr=1e-05, target='age', num_workers=15, weight_decay=1e-05)
+Input dimension: 737280
+total_steps 21750
+wandb_config:
+ {'model_name': 'HCPflat_raw_age', 'batch_size': 256, 'weight_decay': 1e-05, 'num_epochs': 50, 'seed': 42, 'lr_scheduler_type': 'cycle', 'save_ckpt': False, 'max_lr': 1e-05, 'target': 'age', 'num_workers': 15}
+wandb_id: HCPflat_raw_beta_age_18a4fb68-2904-438d-a472-1c5e5f991d72
+Step [100/435] - Training Loss: 0.5783 - Training MSE: 151.1928
+Step [200/435] - Training Loss: 0.5631 - Training MSE: 149.7799
+Step [300/435] - Training Loss: 0.5377 - Training MSE: 148.2206
+Step [400/435] - Training Loss: 0.5397 - Training MSE: 146.6959
+Epoch [1/50] - Training Loss: 0.5718, Training MSE: 146.3073 - Validation Loss: 0.5258, Validation MSE: 134.2583
+Step [100/435] - Training Loss: 0.4520 - Training MSE: 130.8248
+Step [200/435] - Training Loss: 0.5310 - Training MSE: 130.0619
+Step [300/435] - Training Loss: 0.4768 - Training MSE: 128.8698
+Step [400/435] - Training Loss: 0.4999 - Training MSE: 128.3091
+Epoch [2/50] - Training Loss: 0.5001, Training MSE: 127.9771 - Validation Loss: 0.4890, Validation MSE: 124.8453
+Step [100/435] - Training Loss: 0.4212 - Training MSE: 113.6132
+Step [200/435] - Training Loss: 0.4401 - Training MSE: 113.5500
+Step [300/435] - Training Loss: 0.4524 - Training MSE: 113.7848
+Step [400/435] - Training Loss: 0.4064 - Training MSE: 113.7673
+Epoch [3/50] - Training Loss: 0.4440, Training MSE: 113.6280 - Validation Loss: 0.4756, Validation MSE: 121.4521
+Step [100/435] - Training Loss: 0.4088 - Training MSE: 107.9387
+Step [200/435] - Training Loss: 0.4154 - Training MSE: 108.5233
+Step [300/435] - Training Loss: 0.4072 - Training MSE: 108.7730
+Step [400/435] - Training Loss: 0.4568 - Training MSE: 109.0231
+Epoch [4/50] - Training Loss: 0.4260, Training MSE: 109.0222 - Validation Loss: 0.4641, Validation MSE: 118.5301
+Step [100/435] - Training Loss: 0.3889 - Training MSE: 105.9227
+Step [200/435] - Training Loss: 0.4258 - Training MSE: 106.7853
+Step [300/435] - Training Loss: 0.4612 - Training MSE: 107.0059
+Step [400/435] - Training Loss: 0.3901 - Training MSE: 107.3852
+Epoch [5/50] - Training Loss: 0.4196, Training MSE: 107.3848 - Validation Loss: 0.4603, Validation MSE: 117.5609
+Step [100/435] - Training Loss: 0.4250 - Training MSE: 106.4304
+Step [200/435] - Training Loss: 0.4350 - Training MSE: 106.4364
+Step [300/435] - Training Loss: 0.3817 - Training MSE: 106.3877
+Step [400/435] - Training Loss: 0.4380 - Training MSE: 106.7322
+Epoch [6/50] - Training Loss: 0.4172, Training MSE: 106.7691 - Validation Loss: 0.4658, Validation MSE: 118.9548
+Step [100/435] - Training Loss: 0.3975 - Training MSE: 105.6768
+Step [200/435] - Training Loss: 0.3800 - Training MSE: 106.7260
+Step [300/435] - Training Loss: 0.3861 - Training MSE: 106.7522
+Step [400/435] - Training Loss: 0.4227 - Training MSE: 106.7119

541342.err

@@ -0,0 +1,18 @@
+[NbConvertApp] Converting notebook HCP_downstream_finetune.ipynb to python
+[NbConvertApp] Writing 31940 bytes to HCP_downstream_finetune.py
+/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py:658: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
+  state = torch.load(checkpoint_path)
+wandb: Using wandb-core as the SDK backend. Please refer to https://wandb.me/wandb-core for more information.
+wandb: Currently logged in as: ckadirt. Use `wandb login --relogin` to force relogin
+wandb: Tracking run with wandb version 0.18.3
+wandb: Run data is saved locally in /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241127_015634-HCPflat_large_gsrFalse__beta_age_HCPFT_e7c8af61-0ee0-4235-bcdb-bd61bb32c3b5
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run HCPflat_large_gsrFalse__beta_age_HCPFT
+wandb: ⭐️ View project at https://stability.wandb.io/ckadirt/fMRI-foundation-model
+wandb: 🚀 View run at https://stability.wandb.io/ckadirt/fMRI-foundation-model/runs/HCPflat_large_gsrFalse__beta_age_HCPFT_e7c8af61-0ee0-4235-bcdb-bd61bb32c3b5
+
+  return F.mse_loss(input, target, reduction=self.reduction)
+
+slurmstepd: error: *** JOB 541342 ON ip-10-0-136-5 CANCELLED AT 2024-11-27T02:00:31 ***
+slurmstepd: error: *** REASON: burst_buffer/lua: Stage-out in progress ***
+

541342.out

@@ -0,0 +1,68 @@
+NUM_GPUS=1
+MASTER_ADDR=ip-10-0-136-5
+MASTER_PORT=16336
+WORLD_SIZE=1
+------ ARGS ------- 
+ Namespace(found_model_name='HCPflat_large_gsrFalse_', epoch_checkpoint='epoch99.pth', model_suffix='beta', hcp_flat_path='/weka/proj-medarc/shared/HCP-Flat', batch_size=16, wandb_log=True, num_epochs=20, lr_scheduler_type='cycle', save_ckpt=False, seed=42, max_lr=0.0001, target='age', num_workers=10, weight_decay=1e-05, global_pool=True)
+outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_
+Loaded config.yaml from ckpt folder /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_
+
+__CONFIG__
+base_lr = 0.001
+batch_size = 32
+ckpt_interval = 5
+ckpt_saving = True
+cls_embed = True
+contrastive_loss_weight = 1.0
+datasets_to_include = HCP
+decoder_embed_dim = 512
+grad_accumulation_steps = 1
+grad_clip = 1.0
+gsr = False
+hcp_flat_path = /weka/proj-medarc/shared/HCP-Flat
+mask_ratio = 0.75
+model_name = HCPflat_large_gsrFalse_
+no_qkv_bias = False
+norm_pix_loss = False
+nsd_flat_path = /weka/proj-medarc/shared/NSD-Flat
+num_epochs = 100
+num_frames = 16
+num_samples_per_epoch = 200000
+num_workers = 10
+patch_size = 16
+pct_masks_to_decode = 1
+plotting = True
+pred_t_dim = 8
+print_interval = 20
+probe_base_lr = 0.0003
+probe_batch_size = 8
+probe_num_epochs = 30
+probe_num_samples_per_epoch = 100000
+resume_from_ckpt = True
+seed = 42
+sep_pos_embed = True
+t_patch_size = 2
+test_num_samples_per_epoch = 50000
+test_set = False
+trunc_init = False
+use_contrastive_loss = False
+wandb_log = True
+
+
+WORLD_SIZE=1
+PID of this process = 1190785
+global_pool = True
+gsr = False
+Creating datasets
+Datasets ready
+img_size (144, 320) patch_size (16, 16) frames 16 t_patch_size 2
+model initialized
+latest_checkpoint: epoch99.pth
+
+Loaded checkpoint epoch99.pth from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_
+
+Input dimension: 1024
+total_steps 139140
+wandb_config:
+ {'model_name': 'HCPflat_large_gsrFalse__HCP_FT_age', 'batch_size': 16, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42, 'lr_scheduler_type': 'cycle', 'save_ckpt': False, 'max_lr': 0.0001, 'target': 'age', 'num_workers': 10}
+wandb_id: HCPflat_large_gsrFalse__beta_age_HCPFT_e7c8af61-0ee0-4235-bcdb-bd61bb32c3b5

541345.err

@@ -0,0 +1,28 @@
+[NbConvertApp] Converting notebook HCP_downstream_raw_flatmaps.ipynb to python
+[NbConvertApp] Writing 36559 bytes to HCP_downstream_raw_flatmaps.py
+wandb: Using wandb-core as the SDK backend. Please refer to https://wandb.me/wandb-core for more information.
+wandb: Currently logged in as: ckadirt. Use `wandb login --relogin` to force relogin
+wandb: Tracking run with wandb version 0.18.3
+wandb: Run data is saved locally in /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241127_015933-HCPflat_raw_beta_trial_type_3d482888-f4eb-482f-a3c2-2056642d97e2
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run HCPflat_raw_beta_trial_type
+wandb: ⭐️ View project at https://stability.wandb.io/ckadirt/fMRI-foundation-model
+wandb: 🚀 View run at https://stability.wandb.io/ckadirt/fMRI-foundation-model/runs/HCPflat_raw_beta_trial_type_3d482888-f4eb-482f-a3c2-2056642d97e2
+
+Traceback (most recent call last):
+  File "/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_raw_flatmaps.py", line 829, in <module>
+    loss = criterion(outputs, labels)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/admin/home-ckadirt/foundation_env/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1553, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/admin/home-ckadirt/foundation_env/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1562, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/admin/home-ckadirt/foundation_env/lib/python3.11/site-packages/torch/nn/modules/loss.py", line 1188, in forward
+    return F.cross_entropy(input, target, weight=self.weight,
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/admin/home-ckadirt/foundation_env/lib/python3.11/site-packages/torch/nn/functional.py", line 3104, in cross_entropy
+    return torch._C._nn.cross_entropy_loss(input, target, weight, _Reduction.get_enum(reduction), ignore_index, label_smoothing)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+RuntimeError: 0D or 1D target tensor expected, multi-target not supported

541345.out

@@ -0,0 +1,15 @@
+NUM_GPUS=1
+MASTER_ADDR=ip-10-0-135-126
+MASTER_PORT=13292
+WORLD_SIZE=1
+PID of this process = 2152625
+------ ARGS ------- 
+ Namespace(model_suffix='beta', hcp_flat_path='/weka/proj-medarc/shared/HCP-Flat', batch_size=256, wandb_log=True, num_epochs=50, lr_scheduler_type='cycle', save_ckpt=False, seed=42, max_lr=1e-05, target='trial_type', num_workers=15, weight_decay=1e-05)
+Number of classes: 21
+Input dimension: 737280
+total_steps 21750
+wandb_config:
+ {'model_name': 'HCPflat_raw_trial_type', 'batch_size': 256, 'weight_decay': 1e-05, 'num_epochs': 50, 'seed': 42, 'lr_scheduler_type': 'cycle', 'save_ckpt': False, 'max_lr': 1e-05, 'target': 'trial_type', 'num_workers': 15}
+wandb_id: HCPflat_raw_beta_trial_type_3d482888-f4eb-482f-a3c2-2056642d97e2
+wandb: 🚀 View run HCPflat_raw_beta_trial_type at: https://stability.wandb.io/ckadirt/fMRI-foundation-model/runs/HCPflat_raw_beta_trial_type_3d482888-f4eb-482f-a3c2-2056642d97e2
+wandb: Find logs at: wandb/run-20241127_015933-HCPflat_raw_beta_trial_type_3d482888-f4eb-482f-a3c2-2056642d97e2/logs

541349.err

@@ -0,0 +1,18 @@
+[NbConvertApp] Converting notebook HCP_downstream_raw_flatmaps.ipynb to python
+[NbConvertApp] Writing 36673 bytes to HCP_downstream_raw_flatmaps.py
+wandb: Using wandb-core as the SDK backend. Please refer to https://wandb.me/wandb-core for more information.
+wandb: Currently logged in as: ckadirt. Use `wandb login --relogin` to force relogin
+wandb: Tracking run with wandb version 0.18.3
+wandb: Run data is saved locally in /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241127_020911-HCPflat_raw_beta_trial_type_81853367-3038-4b91-805f-5066c048cef4
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run HCPflat_raw_beta_trial_type
+wandb: ⭐️ View project at https://stability.wandb.io/ckadirt/fMRI-foundation-model
+wandb: 🚀 View run at https://stability.wandb.io/ckadirt/fMRI-foundation-model/runs/HCPflat_raw_beta_trial_type_81853367-3038-4b91-805f-5066c048cef4
+
+
+
+
+
+slurmstepd: error: *** JOB 541349 ON ip-10-0-135-126 CANCELLED AT 2024-11-27T02:28:13 ***
+slurmstepd: error: *** REASON: burst_buffer/lua: Stage-out in progress ***
+

541349.out

@@ -0,0 +1,23 @@
+NUM_GPUS=1
+MASTER_ADDR=ip-10-0-135-126
+MASTER_PORT=17074
+WORLD_SIZE=1
+PID of this process = 2161551
+------ ARGS ------- 
+ Namespace(model_suffix='beta', hcp_flat_path='/weka/proj-medarc/shared/HCP-Flat', batch_size=256, wandb_log=True, num_epochs=50, lr_scheduler_type='cycle', save_ckpt=False, seed=42, max_lr=1e-05, target='trial_type', num_workers=15, weight_decay=1e-05)
+Number of classes: 21
+Input dimension: 737280
+total_steps 21750
+wandb_config:
+ {'model_name': 'HCPflat_raw_trial_type', 'batch_size': 256, 'weight_decay': 1e-05, 'num_epochs': 50, 'seed': 42, 'lr_scheduler_type': 'cycle', 'save_ckpt': False, 'max_lr': 1e-05, 'target': 'trial_type', 'num_workers': 15}
+wandb_id: HCPflat_raw_beta_trial_type_81853367-3038-4b91-805f-5066c048cef4
+Step [100/435] - Training Loss: 2.2615 - Training Accuracy: 1472.12%
+Step [200/435] - Training Loss: 1.4896 - Training Accuracy: 1663.83%
+Step [300/435] - Training Loss: 0.8745 - Training Accuracy: 1726.14%
+Step [400/435] - Training Loss: 0.5028 - Training Accuracy: 1742.26%
+Epoch [1/50] - Training Loss: 1.5308, Training Accuracy: 1743.88% - Validation Loss: 0.5084, Validation Accuracy: 1955.07%
+Step [100/435] - Training Loss: 0.2828 - Training Accuracy: 1781.62%
+Step [200/435] - Training Loss: 0.3001 - Training Accuracy: 1787.59%
+Step [300/435] - Training Loss: 0.2278 - Training Accuracy: 1771.23%
+Step [400/435] - Training Loss: 0.2211 - Training Accuracy: 1768.90%
+Epoch [2/50] - Training Loss: 0.2600, Training Accuracy: 1767.59% - Validation Loss: 0.2073, Validation Accuracy: 1961.79%

541350.out

@@ -0,0 +1,264 @@
+NUM_GPUS=1
+MASTER_ADDR=ip-10-0-135-126
+MASTER_PORT=18935
+WORLD_SIZE=1
+PID of this process = 2164521
+------ ARGS ------- 
+ Namespace(model_suffix='beta', hcp_flat_path='/weka/proj-medarc/shared/HCP-Flat', batch_size=256, wandb_log=True, num_epochs=50, lr_scheduler_type='cycle', save_ckpt=False, seed=42, max_lr=1e-05, target='age', num_workers=15, weight_decay=1e-05)
+Input dimension: 737280
+total_steps 21750
+wandb_config:
+ {'model_name': 'HCPflat_raw_age', 'batch_size': 256, 'weight_decay': 1e-05, 'num_epochs': 50, 'seed': 42, 'lr_scheduler_type': 'cycle', 'save_ckpt': False, 'max_lr': 1e-05, 'target': 'age', 'num_workers': 15}
+wandb_id: HCPflat_raw_beta_age_31e54b73-122f-4c96-8d20-21ee38d0705b
+Step [100/435] - Training Loss: 0.6079 - Training MSE: 0.5940
+Step [200/435] - Training Loss: 0.5179 - Training MSE: 0.5895
+Step [300/435] - Training Loss: 0.5806 - Training MSE: 0.5846
+Step [400/435] - Training Loss: 0.4953 - Training MSE: 0.5817
+Epoch [1/50] - Training Loss: 0.5784, Training MSE: 0.5813 - Validation Loss: 0.5355, Validation MSE: 0.5417
+Step [100/435] - Training Loss: 0.4042 - Training MSE: 0.5476
+Step [200/435] - Training Loss: 0.5508 - Training MSE: 0.5488
+Step [300/435] - Training Loss: 0.4799 - Training MSE: 0.5478
+Step [400/435] - Training Loss: 0.5536 - Training MSE: 0.5496
+Epoch [2/50] - Training Loss: 0.4875, Training MSE: 0.5496 - Validation Loss: 0.5427, Validation MSE: 0.5495
+Step [100/435] - Training Loss: 0.3078 - Training MSE: 0.5391
+Step [200/435] - Training Loss: 0.3315 - Training MSE: 0.5413
+Step [300/435] - Training Loss: 0.3964 - Training MSE: 0.5442
+Step [400/435] - Training Loss: 0.3456 - Training MSE: 0.5453
+Epoch [3/50] - Training Loss: 0.3329, Training MSE: 0.5454 - Validation Loss: 0.5490, Validation MSE: 0.5614
+Step [100/435] - Training Loss: 0.2159 - Training MSE: 0.5618
+Step [200/435] - Training Loss: 0.2387 - Training MSE: 0.5638
+Step [300/435] - Training Loss: 0.2004 - Training MSE: 0.5652
+Step [400/435] - Training Loss: 0.2158 - Training MSE: 0.5671
+Epoch [4/50] - Training Loss: 0.2364, Training MSE: 0.5674 - Validation Loss: 0.5676, Validation MSE: 0.5808
+Step [100/435] - Training Loss: 0.1533 - Training MSE: 0.5899
+Step [200/435] - Training Loss: 0.1590 - Training MSE: 0.5937
+Step [300/435] - Training Loss: 0.1987 - Training MSE: 0.5943
+Step [400/435] - Training Loss: 0.2009 - Training MSE: 0.5958
+Epoch [5/50] - Training Loss: 0.1795, Training MSE: 0.5955 - Validation Loss: 0.6034, Validation MSE: 0.6168
+Step [100/435] - Training Loss: 0.1450 - Training MSE: 0.6214
+Step [200/435] - Training Loss: 0.1467 - Training MSE: 0.6192
+Step [300/435] - Training Loss: 0.1742 - Training MSE: 0.6189
+Step [400/435] - Training Loss: 0.1820 - Training MSE: 0.6200
+Epoch [6/50] - Training Loss: 0.1412, Training MSE: 0.6204 - Validation Loss: 0.6164, Validation MSE: 0.6309
+Step [100/435] - Training Loss: 0.0961 - Training MSE: 0.6445
+Step [200/435] - Training Loss: 0.1143 - Training MSE: 0.6474
+Step [300/435] - Training Loss: 0.1104 - Training MSE: 0.6465
+Step [400/435] - Training Loss: 0.1170 - Training MSE: 0.6448
+Epoch [7/50] - Training Loss: 0.1132, Training MSE: 0.6442 - Validation Loss: 0.6420, Validation MSE: 0.6572
+Step [100/435] - Training Loss: 0.0859 - Training MSE: 0.6744
+Step [200/435] - Training Loss: 0.0967 - Training MSE: 0.6682
+Step [300/435] - Training Loss: 0.0962 - Training MSE: 0.6668
+Step [400/435] - Training Loss: 0.1131 - Training MSE: 0.6671
+Epoch [8/50] - Training Loss: 0.0912, Training MSE: 0.6655 - Validation Loss: 0.6658, Validation MSE: 0.6790
+Step [100/435] - Training Loss: 0.0650 - Training MSE: 0.6852
+Step [200/435] - Training Loss: 0.0806 - Training MSE: 0.6835
+Step [300/435] - Training Loss: 0.0789 - Training MSE: 0.6826
+Step [400/435] - Training Loss: 0.0915 - Training MSE: 0.6824
+Epoch [9/50] - Training Loss: 0.0739, Training MSE: 0.6820 - Validation Loss: 0.6844, Validation MSE: 0.7001
+Step [100/435] - Training Loss: 0.0604 - Training MSE: 0.6952
+Step [200/435] - Training Loss: 0.0608 - Training MSE: 0.6973
+Step [300/435] - Training Loss: 0.0601 - Training MSE: 0.6990
+Step [400/435] - Training Loss: 0.0888 - Training MSE: 0.6998
+Epoch [10/50] - Training Loss: 0.0602, Training MSE: 0.6998 - Validation Loss: 0.7134, Validation MSE: 0.7278
+Step [100/435] - Training Loss: 0.0403 - Training MSE: 0.7351
+Step [200/435] - Training Loss: 0.0519 - Training MSE: 0.7232
+Step [300/435] - Training Loss: 0.0490 - Training MSE: 0.7203
+Step [400/435] - Training Loss: 0.0611 - Training MSE: 0.7156
+Epoch [11/50] - Training Loss: 0.0492, Training MSE: 0.7141 - Validation Loss: 0.7246, Validation MSE: 0.7388
+Step [100/435] - Training Loss: 0.0310 - Training MSE: 0.7381
+Step [200/435] - Training Loss: 0.0368 - Training MSE: 0.7302
+Step [300/435] - Training Loss: 0.0446 - Training MSE: 0.7282
+Step [400/435] - Training Loss: 0.0474 - Training MSE: 0.7276
+Epoch [12/50] - Training Loss: 0.0400, Training MSE: 0.7267 - Validation Loss: 0.7409, Validation MSE: 0.7569
+Step [100/435] - Training Loss: 0.0315 - Training MSE: 0.7505
+Step [200/435] - Training Loss: 0.0310 - Training MSE: 0.7421
+Step [300/435] - Training Loss: 0.0356 - Training MSE: 0.7409
+Step [400/435] - Training Loss: 0.0428 - Training MSE: 0.7382
+Epoch [13/50] - Training Loss: 0.0324, Training MSE: 0.7377 - Validation Loss: 0.7593, Validation MSE: 0.7739
+Step [100/435] - Training Loss: 0.0229 - Training MSE: 0.7523
+Step [200/435] - Training Loss: 0.0279 - Training MSE: 0.7534
+Step [300/435] - Training Loss: 0.0317 - Training MSE: 0.7516
+Step [400/435] - Training Loss: 0.0314 - Training MSE: 0.7493
+Epoch [14/50] - Training Loss: 0.0268, Training MSE: 0.7477 - Validation Loss: 0.7724, Validation MSE: 0.7878
+Step [100/435] - Training Loss: 0.0163 - Training MSE: 0.7665
+Step [200/435] - Training Loss: 0.0220 - Training MSE: 0.7628
+Step [300/435] - Training Loss: 0.0242 - Training MSE: 0.7608
+Step [400/435] - Training Loss: 0.0288 - Training MSE: 0.7579
+Epoch [15/50] - Training Loss: 0.0215, Training MSE: 0.7572 - Validation Loss: 0.7895, Validation MSE: 0.8040
+Step [100/435] - Training Loss: 0.0134 - Training MSE: 0.7748
+Step [200/435] - Training Loss: 0.0160 - Training MSE: 0.7727
+Step [300/435] - Training Loss: 0.0197 - Training MSE: 0.7660
+Step [400/435] - Training Loss: 0.0230 - Training MSE: 0.7650
+Epoch [16/50] - Training Loss: 0.0180, Training MSE: 0.7649 - Validation Loss: 0.8056, Validation MSE: 0.8198
+Step [100/435] - Training Loss: 0.0127 - Training MSE: 0.7814
+Step [200/435] - Training Loss: 0.0157 - Training MSE: 0.7747
+Step [300/435] - Training Loss: 0.0165 - Training MSE: 0.7720
+Step [400/435] - Training Loss: 0.0159 - Training MSE: 0.7728
+Epoch [17/50] - Training Loss: 0.0145, Training MSE: 0.7717 - Validation Loss: 0.8114, Validation MSE: 0.8293
+Step [100/435] - Training Loss: 0.0095 - Training MSE: 0.7832
+Step [200/435] - Training Loss: 0.0129 - Training MSE: 0.7875
+Step [300/435] - Training Loss: 0.0104 - Training MSE: 0.7807
+Step [400/435] - Training Loss: 0.0119 - Training MSE: 0.7786
+Epoch [18/50] - Training Loss: 0.0119, Training MSE: 0.7773 - Validation Loss: 0.8136, Validation MSE: 0.8289
+Step [100/435] - Training Loss: 0.0080 - Training MSE: 0.7895
+Step [200/435] - Training Loss: 0.0097 - Training MSE: 0.7878
+Step [300/435] - Training Loss: 0.0100 - Training MSE: 0.7860
+Step [400/435] - Training Loss: 0.0121 - Training MSE: 0.7824
+Epoch [19/50] - Training Loss: 0.0100, Training MSE: 0.7826 - Validation Loss: 0.8201, Validation MSE: 0.8367
+Step [100/435] - Training Loss: 0.0101 - Training MSE: 0.7886
+Step [200/435] - Training Loss: 0.0097 - Training MSE: 0.7900
+Step [300/435] - Training Loss: 0.0096 - Training MSE: 0.7883
+Step [400/435] - Training Loss: 0.0120 - Training MSE: 0.7878
+Epoch [20/50] - Training Loss: 0.0084, Training MSE: 0.7878 - Validation Loss: 0.8246, Validation MSE: 0.8405
+Step [100/435] - Training Loss: 0.0061 - Training MSE: 0.7964
+Step [200/435] - Training Loss: 0.0061 - Training MSE: 0.7913
+Step [300/435] - Training Loss: 0.0081 - Training MSE: 0.7896
+Step [400/435] - Training Loss: 0.0067 - Training MSE: 0.7924
+Epoch [21/50] - Training Loss: 0.0071, Training MSE: 0.7912 - Validation Loss: 0.8344, Validation MSE: 0.8493
+Step [100/435] - Training Loss: 0.0067 - Training MSE: 0.8002
+Step [200/435] - Training Loss: 0.0071 - Training MSE: 0.8009
+Step [300/435] - Training Loss: 0.0070 - Training MSE: 0.7963
+Step [400/435] - Training Loss: 0.0053 - Training MSE: 0.7946
+Epoch [22/50] - Training Loss: 0.0060, Training MSE: 0.7941 - Validation Loss: 0.8400, Validation MSE: 0.8555
+Step [100/435] - Training Loss: 0.0046 - Training MSE: 0.7930
+Step [200/435] - Training Loss: 0.0054 - Training MSE: 0.7987
+Step [300/435] - Training Loss: 0.0048 - Training MSE: 0.7986
+Step [400/435] - Training Loss: 0.0058 - Training MSE: 0.7976
+Epoch [23/50] - Training Loss: 0.0051, Training MSE: 0.7968 - Validation Loss: 0.8398, Validation MSE: 0.8552
+Step [100/435] - Training Loss: 0.0046 - Training MSE: 0.8091
+Step [200/435] - Training Loss: 0.0044 - Training MSE: 0.8047
+Step [300/435] - Training Loss: 0.0041 - Training MSE: 0.7978
+Step [400/435] - Training Loss: 0.0041 - Training MSE: 0.7975
+Epoch [24/50] - Training Loss: 0.0046, Training MSE: 0.7982 - Validation Loss: 0.8454, Validation MSE: 0.8614
+Step [100/435] - Training Loss: 0.0043 - Training MSE: 0.8051
+Step [200/435] - Training Loss: 0.0041 - Training MSE: 0.8029
+Step [300/435] - Training Loss: 0.0036 - Training MSE: 0.8008
+Step [400/435] - Training Loss: 0.0034 - Training MSE: 0.8006
+Epoch [25/50] - Training Loss: 0.0040, Training MSE: 0.8000 - Validation Loss: 0.8471, Validation MSE: 0.8635
+Step [100/435] - Training Loss: 0.0035 - Training MSE: 0.7997
+Step [200/435] - Training Loss: 0.0028 - Training MSE: 0.8036
+Step [300/435] - Training Loss: 0.0036 - Training MSE: 0.8002
+Step [400/435] - Training Loss: 0.0042 - Training MSE: 0.8029
+Epoch [26/50] - Training Loss: 0.0037, Training MSE: 0.8015 - Validation Loss: 0.8479, Validation MSE: 0.8643
+Step [100/435] - Training Loss: 0.0035 - Training MSE: 0.8106
+Step [200/435] - Training Loss: 0.0054 - Training MSE: 0.8063
+Step [300/435] - Training Loss: 0.0046 - Training MSE: 0.8047
+Step [400/435] - Training Loss: 0.0040 - Training MSE: 0.8028
+Epoch [27/50] - Training Loss: 0.0043, Training MSE: 0.8031 - Validation Loss: 0.8483, Validation MSE: 0.8642
+Step [100/435] - Training Loss: 0.0028 - Training MSE: 0.8015
+Step [200/435] - Training Loss: 0.0040 - Training MSE: 0.8030
+Step [300/435] - Training Loss: 0.0036 - Training MSE: 0.8030
+Step [400/435] - Training Loss: 0.0079 - Training MSE: 0.8025
+Epoch [28/50] - Training Loss: 0.0037, Training MSE: 0.8037 - Validation Loss: 0.8482, Validation MSE: 0.8644
+Step [100/435] - Training Loss: 0.0133 - Training MSE: 0.8092
+Step [200/435] - Training Loss: 0.0036 - Training MSE: 0.8067
+Step [300/435] - Training Loss: 0.0033 - Training MSE: 0.8063
+Step [400/435] - Training Loss: 0.0020 - Training MSE: 0.8067
+Epoch [29/50] - Training Loss: 0.0044, Training MSE: 0.8054 - Validation Loss: 0.8503, Validation MSE: 0.8654
+Step [100/435] - Training Loss: 0.0021 - Training MSE: 0.8117
+Step [200/435] - Training Loss: 0.0121 - Training MSE: 0.8105
+Step [300/435] - Training Loss: 0.0028 - Training MSE: 0.8060
+Step [400/435] - Training Loss: 0.0025 - Training MSE: 0.8052
+Epoch [30/50] - Training Loss: 0.0038, Training MSE: 0.8053 - Validation Loss: 0.8541, Validation MSE: 0.8688
+Step [100/435] - Training Loss: 0.0014 - Training MSE: 0.7938
+Step [200/435] - Training Loss: 0.0031 - Training MSE: 0.8074
+Step [300/435] - Training Loss: 0.0015 - Training MSE: 0.8058
+Step [400/435] - Training Loss: 0.0017 - Training MSE: 0.8041
+Epoch [31/50] - Training Loss: 0.0025, Training MSE: 0.8045 - Validation Loss: 0.8527, Validation MSE: 0.8680
+Step [100/435] - Training Loss: 0.0031 - Training MSE: 0.8159
+Step [200/435] - Training Loss: 0.0014 - Training MSE: 0.8139
+Step [300/435] - Training Loss: 0.0012 - Training MSE: 0.8103
+Step [400/435] - Training Loss: 0.0011 - Training MSE: 0.8056
+Epoch [32/50] - Training Loss: 0.0019, Training MSE: 0.8038 - Validation Loss: 0.8513, Validation MSE: 0.8665
+Step [100/435] - Training Loss: 0.0102 - Training MSE: 0.8013
+Step [200/435] - Training Loss: 0.0009 - Training MSE: 0.8001
+Step [300/435] - Training Loss: 0.0006 - Training MSE: 0.8021
+Step [400/435] - Training Loss: 0.0008 - Training MSE: 0.8021
+Epoch [33/50] - Training Loss: 0.0008, Training MSE: 0.8031 - Validation Loss: 0.8515, Validation MSE: 0.8670
+Step [100/435] - Training Loss: 0.0004 - Training MSE: 0.7997
+Step [200/435] - Training Loss: 0.0004 - Training MSE: 0.8031
+Step [300/435] - Training Loss: 0.0004 - Training MSE: 0.8047
+Step [400/435] - Training Loss: 0.0005 - Training MSE: 0.8031
+Epoch [34/50] - Training Loss: 0.0004, Training MSE: 0.8029 - Validation Loss: 0.8521, Validation MSE: 0.8676
+Step [100/435] - Training Loss: 0.0002 - Training MSE: 0.7941
+Step [200/435] - Training Loss: 0.0005 - Training MSE: 0.7996
+Step [300/435] - Training Loss: 0.0003 - Training MSE: 0.8016
+Step [400/435] - Training Loss: 0.0003 - Training MSE: 0.8022
+Epoch [35/50] - Training Loss: 0.0003, Training MSE: 0.8028 - Validation Loss: 0.8521, Validation MSE: 0.8676
+Step [100/435] - Training Loss: 0.0006 - Training MSE: 0.8030
+Step [200/435] - Training Loss: 0.0002 - Training MSE: 0.8033
+Step [300/435] - Training Loss: 0.0002 - Training MSE: 0.8046
+Step [400/435] - Training Loss: 0.0002 - Training MSE: 0.8030
+Epoch [36/50] - Training Loss: 0.0002, Training MSE: 0.8028 - Validation Loss: 0.8526, Validation MSE: 0.8683
+Step [100/435] - Training Loss: 0.0001 - Training MSE: 0.8003
+Step [200/435] - Training Loss: 0.0002 - Training MSE: 0.8045
+Step [300/435] - Training Loss: 0.0002 - Training MSE: 0.8042
+Step [400/435] - Training Loss: 0.0002 - Training MSE: 0.8047
+Epoch [37/50] - Training Loss: 0.0002, Training MSE: 0.8033 - Validation Loss: 0.8518, Validation MSE: 0.8675
+Step [100/435] - Training Loss: 0.0001 - Training MSE: 0.8073
+Step [200/435] - Training Loss: 0.0002 - Training MSE: 0.8066
+Step [300/435] - Training Loss: 0.0001 - Training MSE: 0.8049
+Step [400/435] - Training Loss: 0.0001 - Training MSE: 0.8029
+Epoch [38/50] - Training Loss: 0.0002, Training MSE: 0.8030 - Validation Loss: 0.8526, Validation MSE: 0.8680
+Step [100/435] - Training Loss: 0.0001 - Training MSE: 0.8039
+Step [200/435] - Training Loss: 0.0002 - Training MSE: 0.7996
+Step [300/435] - Training Loss: 0.0001 - Training MSE: 0.8031
+Step [400/435] - Training Loss: 0.0001 - Training MSE: 0.8036
+Epoch [39/50] - Training Loss: 0.0001, Training MSE: 0.8035 - Validation Loss: 0.8525, Validation MSE: 0.8681
+Step [100/435] - Training Loss: 0.0001 - Training MSE: 0.8010
+Step [200/435] - Training Loss: 0.0001 - Training MSE: 0.8027
+Step [300/435] - Training Loss: 0.0002 - Training MSE: 0.8015
+Step [400/435] - Training Loss: 0.0001 - Training MSE: 0.8031
+Epoch [40/50] - Training Loss: 0.0001, Training MSE: 0.8036 - Validation Loss: 0.8533, Validation MSE: 0.8687
+Step [100/435] - Training Loss: 0.0001 - Training MSE: 0.8101
+Step [200/435] - Training Loss: 0.0001 - Training MSE: 0.8026
+Step [300/435] - Training Loss: 0.0001 - Training MSE: 0.8033
+Step [400/435] - Training Loss: 0.0001 - Training MSE: 0.8038
+Epoch [41/50] - Training Loss: 0.0001, Training MSE: 0.8033 - Validation Loss: 0.8533, Validation MSE: 0.8688
+Step [100/435] - Training Loss: 0.0001 - Training MSE: 0.8005
+Step [200/435] - Training Loss: 0.0001 - Training MSE: 0.8006
+Step [300/435] - Training Loss: 0.0000 - Training MSE: 0.8055
+Step [400/435] - Training Loss: 0.0000 - Training MSE: 0.8031
+Epoch [42/50] - Training Loss: 0.0001, Training MSE: 0.8035 - Validation Loss: 0.8533, Validation MSE: 0.8689
+Step [100/435] - Training Loss: 0.0000 - Training MSE: 0.7973
+Step [200/435] - Training Loss: 0.0000 - Training MSE: 0.8005
+Step [300/435] - Training Loss: 0.0000 - Training MSE: 0.8027
+Step [400/435] - Training Loss: 0.0000 - Training MSE: 0.8034
+Epoch [43/50] - Training Loss: 0.0000, Training MSE: 0.8033 - Validation Loss: 0.8532, Validation MSE: 0.8688
+Step [100/435] - Training Loss: 0.0000 - Training MSE: 0.7967
+Step [200/435] - Training Loss: 0.0000 - Training MSE: 0.7960
+Step [300/435] - Training Loss: 0.0000 - Training MSE: 0.7976
+Step [400/435] - Training Loss: 0.0000 - Training MSE: 0.8024
+Epoch [44/50] - Training Loss: 0.0000, Training MSE: 0.8032 - Validation Loss: 0.8533, Validation MSE: 0.8688
+Step [100/435] - Training Loss: 0.0000 - Training MSE: 0.8043
+Step [200/435] - Training Loss: 0.0000 - Training MSE: 0.8054
+Step [300/435] - Training Loss: 0.0000 - Training MSE: 0.8052
+Step [400/435] - Training Loss: 0.0000 - Training MSE: 0.8047
+Epoch [45/50] - Training Loss: 0.0000, Training MSE: 0.8037 - Validation Loss: 0.8533, Validation MSE: 0.8689
+Step [100/435] - Training Loss: 0.0000 - Training MSE: 0.8019
+Step [200/435] - Training Loss: 0.0000 - Training MSE: 0.8023
+Step [300/435] - Training Loss: 0.0000 - Training MSE: 0.8026
+Step [400/435] - Training Loss: 0.0000 - Training MSE: 0.8041
+Epoch [46/50] - Training Loss: 0.0000, Training MSE: 0.8032 - Validation Loss: 0.8533, Validation MSE: 0.8689
+Step [100/435] - Training Loss: 0.0000 - Training MSE: 0.7993
+Step [200/435] - Training Loss: 0.0000 - Training MSE: 0.8029
+Step [300/435] - Training Loss: 0.0000 - Training MSE: 0.8066
+Step [400/435] - Training Loss: 0.0000 - Training MSE: 0.8051
+Epoch [47/50] - Training Loss: 0.0000, Training MSE: 0.8036 - Validation Loss: 0.8533, Validation MSE: 0.8689
+Step [100/435] - Training Loss: 0.0000 - Training MSE: 0.7992
+Step [200/435] - Training Loss: 0.0000 - Training MSE: 0.8020
+Step [300/435] - Training Loss: 0.0000 - Training MSE: 0.8030
+Step [400/435] - Training Loss: 0.0000 - Training MSE: 0.8036
+Epoch [48/50] - Training Loss: 0.0000, Training MSE: 0.8032 - Validation Loss: 0.8533, Validation MSE: 0.8689
+Step [100/435] - Training Loss: 0.0000 - Training MSE: 0.7972
+Step [200/435] - Training Loss: 0.0000 - Training MSE: 0.7993
+Step [300/435] - Training Loss: 0.0000 - Training MSE: 0.8023
+Step [400/435] - Training Loss: 0.0000 - Training MSE: 0.8016
+Epoch [49/50] - Training Loss: 0.0000, Training MSE: 0.8035 - Validation Loss: 0.8533, Validation MSE: 0.8689
+Step [100/435] - Training Loss: 0.0000 - Training MSE: 0.8021
+Step [200/435] - Training Loss: 0.0000 - Training MSE: 0.8040
+Step [300/435] - Training Loss: 0.0000 - Training MSE: 0.8023
+Step [400/435] - Training Loss: 0.0000 - Training MSE: 0.8036
+Epoch [50/50] - Training Loss: 0.0000, Training MSE: 0.8037 - Validation Loss: 0.8533, Validation MSE: 0.8689
+wandb: 🚀 View run HCPflat_raw_beta_age at: https://stability.wandb.io/ckadirt/fMRI-foundation-model/runs/HCPflat_raw_beta_age_31e54b73-122f-4c96-8d20-21ee38d0705b
+wandb: Find logs at: wandb/run-20241127_021238-HCPflat_raw_beta_age_31e54b73-122f-4c96-8d20-21ee38d0705b/logs

541355.out

@@ -0,0 +1,265 @@
+NUM_GPUS=1
+MASTER_ADDR=ip-10-0-131-135
+MASTER_PORT=13958
+WORLD_SIZE=1
+PID of this process = 1419057
+------ ARGS ------- 
+ Namespace(model_suffix='beta', hcp_flat_path='/weka/proj-medarc/shared/HCP-Flat', batch_size=256, wandb_log=True, num_epochs=50, lr_scheduler_type='cycle', save_ckpt=False, seed=42, max_lr=1e-05, target='trial_type', num_workers=15, weight_decay=1e-05)
+Number of classes: 21
+Input dimension: 737280
+total_steps 21750
+wandb_config:
+ {'model_name': 'HCPflat_raw_trial_type', 'batch_size': 256, 'weight_decay': 1e-05, 'num_epochs': 50, 'seed': 42, 'lr_scheduler_type': 'cycle', 'save_ckpt': False, 'max_lr': 1e-05, 'target': 'trial_type', 'num_workers': 15}
+wandb_id: HCPflat_raw_beta_trial_type_749d6203-1266-473a-b49a-e7325917f171
+Step [100/435] - Training Loss: 2.2615 - Training Accuracy: 21.61%
+Step [200/435] - Training Loss: 1.4896 - Training Accuracy: 38.51%
+Step [300/435] - Training Loss: 0.8745 - Training Accuracy: 50.88%
+Step [400/435] - Training Loss: 0.5028 - Training Accuracy: 59.75%
+Epoch [1/50] - Training Loss: 1.5308, Training Accuracy: 62.16% - Validation Loss: 0.5084, Validation Accuracy: 90.71%
+Step [100/435] - Training Loss: 0.2828 - Training Accuracy: 94.09%
+Step [200/435] - Training Loss: 0.3001 - Training Accuracy: 94.64%
+Step [300/435] - Training Loss: 0.2278 - Training Accuracy: 95.04%
+Step [400/435] - Training Loss: 0.2211 - Training Accuracy: 95.40%
+Epoch [2/50] - Training Loss: 0.2600, Training Accuracy: 95.54% - Validation Loss: 0.2073, Validation Accuracy: 95.64%
+Step [100/435] - Training Loss: 0.0558 - Training Accuracy: 99.27%
+Step [200/435] - Training Loss: 0.0562 - Training Accuracy: 99.29%
+Step [300/435] - Training Loss: 0.0575 - Training Accuracy: 99.27%
+Step [400/435] - Training Loss: 0.0635 - Training Accuracy: 99.29%
+Epoch [3/50] - Training Loss: 0.0729, Training Accuracy: 99.28% - Validation Loss: 0.1615, Validation Accuracy: 96.23%
+Step [100/435] - Training Loss: 0.0202 - Training Accuracy: 99.91%
+Step [200/435] - Training Loss: 0.0259 - Training Accuracy: 99.91%
+Step [300/435] - Training Loss: 0.0311 - Training Accuracy: 99.91%
+Step [400/435] - Training Loss: 0.0316 - Training Accuracy: 99.91%
+Epoch [4/50] - Training Loss: 0.0291, Training Accuracy: 99.91% - Validation Loss: 0.1442, Validation Accuracy: 96.42%
+Step [100/435] - Training Loss: 0.0156 - Training Accuracy: 99.98%
+Step [200/435] - Training Loss: 0.0134 - Training Accuracy: 99.98%
+Step [300/435] - Training Loss: 0.0142 - Training Accuracy: 99.98%
+Step [400/435] - Training Loss: 0.0202 - Training Accuracy: 99.98%
+Epoch [5/50] - Training Loss: 0.0157, Training Accuracy: 99.98% - Validation Loss: 0.1360, Validation Accuracy: 96.47%
+Step [100/435] - Training Loss: 0.0093 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0101 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0113 - Training Accuracy: 99.99%
+Step [400/435] - Training Loss: 0.0102 - Training Accuracy: 99.99%
+Epoch [6/50] - Training Loss: 0.0099, Training Accuracy: 99.99% - Validation Loss: 0.1300, Validation Accuracy: 96.64%
+Step [100/435] - Training Loss: 0.0074 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0065 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0066 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0058 - Training Accuracy: 100.00%
+Epoch [7/50] - Training Loss: 0.0067, Training Accuracy: 100.00% - Validation Loss: 0.1265, Validation Accuracy: 96.62%
+Step [100/435] - Training Loss: 0.0050 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0063 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0040 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0055 - Training Accuracy: 100.00%
+Epoch [8/50] - Training Loss: 0.0049, Training Accuracy: 100.00% - Validation Loss: 0.1237, Validation Accuracy: 96.67%
+Step [100/435] - Training Loss: 0.0040 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0036 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0033 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0027 - Training Accuracy: 100.00%
+Epoch [9/50] - Training Loss: 0.0037, Training Accuracy: 100.00% - Validation Loss: 0.1222, Validation Accuracy: 96.64%
+Step [100/435] - Training Loss: 0.0030 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0029 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0030 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0022 - Training Accuracy: 100.00%
+Epoch [10/50] - Training Loss: 0.0028, Training Accuracy: 100.00% - Validation Loss: 0.1203, Validation Accuracy: 96.71%
+Step [100/435] - Training Loss: 0.0024 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0020 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0019 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0023 - Training Accuracy: 100.00%
+Epoch [11/50] - Training Loss: 0.0022, Training Accuracy: 100.00% - Validation Loss: 0.1192, Validation Accuracy: 96.71%
+Step [100/435] - Training Loss: 0.0019 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0015 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0016 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0018 - Training Accuracy: 100.00%
+Epoch [12/50] - Training Loss: 0.0017, Training Accuracy: 100.00% - Validation Loss: 0.1184, Validation Accuracy: 96.78%
+Step [100/435] - Training Loss: 0.0014 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0013 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0014 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0013 - Training Accuracy: 100.00%
+Epoch [13/50] - Training Loss: 0.0014, Training Accuracy: 100.00% - Validation Loss: 0.1174, Validation Accuracy: 96.76%
+Step [100/435] - Training Loss: 0.0011 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0011 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0012 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0012 - Training Accuracy: 100.00%
+Epoch [14/50] - Training Loss: 0.0011, Training Accuracy: 100.00% - Validation Loss: 0.1170, Validation Accuracy: 96.84%
+Step [100/435] - Training Loss: 0.0009 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0008 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0008 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0009 - Training Accuracy: 100.00%
+Epoch [15/50] - Training Loss: 0.0009, Training Accuracy: 100.00% - Validation Loss: 0.1167, Validation Accuracy: 96.81%
+Step [100/435] - Training Loss: 0.0008 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0007 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0006 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0006 - Training Accuracy: 100.00%
+Epoch [16/50] - Training Loss: 0.0007, Training Accuracy: 100.00% - Validation Loss: 0.1167, Validation Accuracy: 96.82%
+Step [100/435] - Training Loss: 0.0006 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0006 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0007 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0006 - Training Accuracy: 100.00%
+Epoch [17/50] - Training Loss: 0.0006, Training Accuracy: 100.00% - Validation Loss: 0.1166, Validation Accuracy: 96.83%
+Step [100/435] - Training Loss: 0.0005 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0005 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0004 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0005 - Training Accuracy: 100.00%
+Epoch [18/50] - Training Loss: 0.0005, Training Accuracy: 100.00% - Validation Loss: 0.1164, Validation Accuracy: 96.82%
+Step [100/435] - Training Loss: 0.0003 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0004 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0004 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0003 - Training Accuracy: 100.00%
+Epoch [19/50] - Training Loss: 0.0004, Training Accuracy: 100.00% - Validation Loss: 0.1162, Validation Accuracy: 96.84%
+Step [100/435] - Training Loss: 0.0003 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0003 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0003 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0003 - Training Accuracy: 100.00%
+Epoch [20/50] - Training Loss: 0.0003, Training Accuracy: 100.00% - Validation Loss: 0.1163, Validation Accuracy: 96.87%
+Step [100/435] - Training Loss: 0.0003 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0003 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0003 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0002 - Training Accuracy: 100.00%
+Epoch [21/50] - Training Loss: 0.0003, Training Accuracy: 100.00% - Validation Loss: 0.1169, Validation Accuracy: 96.87%
+Step [100/435] - Training Loss: 0.0003 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0002 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0002 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0002 - Training Accuracy: 100.00%
+Epoch [22/50] - Training Loss: 0.0002, Training Accuracy: 100.00% - Validation Loss: 0.1164, Validation Accuracy: 96.87%
+Step [100/435] - Training Loss: 0.0002 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0002 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0002 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0002 - Training Accuracy: 100.00%
+Epoch [23/50] - Training Loss: 0.0002, Training Accuracy: 100.00% - Validation Loss: 0.1171, Validation Accuracy: 96.90%
+Step [100/435] - Training Loss: 0.0002 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0002 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0002 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0002 - Training Accuracy: 100.00%
+Epoch [24/50] - Training Loss: 0.0002, Training Accuracy: 100.00% - Validation Loss: 0.1170, Validation Accuracy: 96.92%
+Step [100/435] - Training Loss: 0.0001 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0001 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0001 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0001 - Training Accuracy: 100.00%
+Epoch [25/50] - Training Loss: 0.0001, Training Accuracy: 100.00% - Validation Loss: 0.1177, Validation Accuracy: 96.93%
+Step [100/435] - Training Loss: 0.0001 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0001 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0001 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0001 - Training Accuracy: 100.00%
+Epoch [26/50] - Training Loss: 0.0001, Training Accuracy: 100.00% - Validation Loss: 0.1178, Validation Accuracy: 96.91%
+Step [100/435] - Training Loss: 0.0001 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0001 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0001 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0001 - Training Accuracy: 100.00%
+Epoch [27/50] - Training Loss: 0.0001, Training Accuracy: 100.00% - Validation Loss: 0.1180, Validation Accuracy: 96.93%
+Step [100/435] - Training Loss: 0.0001 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0001 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0001 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0001 - Training Accuracy: 100.00%
+Epoch [28/50] - Training Loss: 0.0001, Training Accuracy: 100.00% - Validation Loss: 0.1183, Validation Accuracy: 96.92%
+Step [100/435] - Training Loss: 0.0001 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0001 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0001 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0001 - Training Accuracy: 100.00%
+Epoch [29/50] - Training Loss: 0.0001, Training Accuracy: 100.00% - Validation Loss: 0.1192, Validation Accuracy: 96.94%
+Step [100/435] - Training Loss: 0.0001 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0001 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0001 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0001 - Training Accuracy: 100.00%
+Epoch [30/50] - Training Loss: 0.0001, Training Accuracy: 100.00% - Validation Loss: 0.1193, Validation Accuracy: 96.95%
+Step [100/435] - Training Loss: 0.0001 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0001 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0001 - Training Accuracy: 100.00%
+Epoch [31/50] - Training Loss: 0.0000, Training Accuracy: 100.00% - Validation Loss: 0.1198, Validation Accuracy: 96.96%
+Step [100/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Epoch [32/50] - Training Loss: 0.0000, Training Accuracy: 100.00% - Validation Loss: 0.1203, Validation Accuracy: 96.94%
+Step [100/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Epoch [33/50] - Training Loss: 0.0000, Training Accuracy: 100.00% - Validation Loss: 0.1206, Validation Accuracy: 96.97%
+Step [100/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Epoch [34/50] - Training Loss: 0.0000, Training Accuracy: 100.00% - Validation Loss: 0.1208, Validation Accuracy: 96.98%
+Step [100/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Epoch [35/50] - Training Loss: 0.0000, Training Accuracy: 100.00% - Validation Loss: 0.1214, Validation Accuracy: 96.99%
+Step [100/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Epoch [36/50] - Training Loss: 0.0000, Training Accuracy: 100.00% - Validation Loss: 0.1220, Validation Accuracy: 97.00%
+Step [100/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Epoch [37/50] - Training Loss: 0.0000, Training Accuracy: 100.00% - Validation Loss: 0.1224, Validation Accuracy: 96.99%
+Step [100/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Epoch [38/50] - Training Loss: 0.0000, Training Accuracy: 100.00% - Validation Loss: 0.1229, Validation Accuracy: 97.00%
+Step [100/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Epoch [39/50] - Training Loss: 0.0000, Training Accuracy: 100.00% - Validation Loss: 0.1232, Validation Accuracy: 97.00%
+Step [100/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Epoch [40/50] - Training Loss: 0.0000, Training Accuracy: 100.00% - Validation Loss: 0.1235, Validation Accuracy: 97.00%
+Step [100/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Epoch [41/50] - Training Loss: 0.0000, Training Accuracy: 100.00% - Validation Loss: 0.1239, Validation Accuracy: 97.01%
+Step [100/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Epoch [42/50] - Training Loss: 0.0000, Training Accuracy: 100.00% - Validation Loss: 0.1242, Validation Accuracy: 97.02%
+Step [100/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Epoch [43/50] - Training Loss: 0.0000, Training Accuracy: 100.00% - Validation Loss: 0.1244, Validation Accuracy: 97.00%
+Step [100/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Epoch [44/50] - Training Loss: 0.0000, Training Accuracy: 100.00% - Validation Loss: 0.1247, Validation Accuracy: 97.02%
+Step [100/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Epoch [45/50] - Training Loss: 0.0000, Training Accuracy: 100.00% - Validation Loss: 0.1250, Validation Accuracy: 97.03%
+Step [100/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Epoch [46/50] - Training Loss: 0.0000, Training Accuracy: 100.00% - Validation Loss: 0.1252, Validation Accuracy: 97.02%
+Step [100/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Epoch [47/50] - Training Loss: 0.0000, Training Accuracy: 100.00% - Validation Loss: 0.1252, Validation Accuracy: 97.04%
+Step [100/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Epoch [48/50] - Training Loss: 0.0000, Training Accuracy: 100.00% - Validation Loss: 0.1253, Validation Accuracy: 97.04%
+Step [100/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Epoch [49/50] - Training Loss: 0.0000, Training Accuracy: 100.00% - Validation Loss: 0.1253, Validation Accuracy: 97.04%
+Step [100/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [200/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [300/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Step [400/435] - Training Loss: 0.0000 - Training Accuracy: 100.00%
+Epoch [50/50] - Training Loss: 0.0000, Training Accuracy: 100.00% - Validation Loss: 0.1253, Validation Accuracy: 97.04%
+wandb: 🚀 View run HCPflat_raw_beta_trial_type at: https://stability.wandb.io/ckadirt/fMRI-foundation-model/runs/HCPflat_raw_beta_trial_type_749d6203-1266-473a-b49a-e7325917f171
+wandb: Find logs at: wandb/run-20241127_023019-HCPflat_raw_beta_trial_type_749d6203-1266-473a-b49a-e7325917f171/logs

541357.err

@@ -0,0 +1,15 @@
+[NbConvertApp] Converting notebook HCP_downstream_finetune.ipynb to python
+[NbConvertApp] Writing 31964 bytes to HCP_downstream_finetune.py
+/weka/proj-fmri/ckadirt/fMRI-foundation-model/src/HCP_downstream_finetune.py:658: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
+  state = torch.load(checkpoint_path)
+wandb: Using wandb-core as the SDK backend. Please refer to https://wandb.me/wandb-core for more information.
+wandb: Currently logged in as: ckadirt. Use `wandb login --relogin` to force relogin
+wandb: Tracking run with wandb version 0.18.3
+wandb: Run data is saved locally in /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/wandb/run-20241127_023444-HCPflat_large_gsrFalse__beta_age_HCPFT_768d7c19-0891-45ae-8cff-9b10e484c8fe
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run HCPflat_large_gsrFalse__beta_age_HCPFT
+wandb: ⭐️ View project at https://stability.wandb.io/ckadirt/fMRI-foundation-model
+wandb: 🚀 View run at https://stability.wandb.io/ckadirt/fMRI-foundation-model/runs/HCPflat_large_gsrFalse__beta_age_HCPFT_768d7c19-0891-45ae-8cff-9b10e484c8fe
+
+slurmstepd: error: *** JOB 541357 ON ip-10-0-135-126 CANCELLED AT 2024-11-27T02:34:48 ***
+slurmstepd: error: *** REASON: burst_buffer/lua: Stage-out in progress ***

541357.out

@@ -0,0 +1,68 @@
+NUM_GPUS=1
+MASTER_ADDR=ip-10-0-135-126
+MASTER_PORT=11002
+WORLD_SIZE=1
+------ ARGS ------- 
+ Namespace(found_model_name='HCPflat_large_gsrFalse_', epoch_checkpoint='epoch99.pth', model_suffix='beta', hcp_flat_path='/weka/proj-medarc/shared/HCP-Flat', batch_size=16, wandb_log=True, num_epochs=20, lr_scheduler_type='cycle', save_ckpt=False, seed=42, max_lr=0.0001, target='age', num_workers=10, weight_decay=1e-05, global_pool=True)
+outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_
+Loaded config.yaml from ckpt folder /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_
+
+__CONFIG__
+base_lr = 0.001
+batch_size = 32
+ckpt_interval = 5
+ckpt_saving = True
+cls_embed = True
+contrastive_loss_weight = 1.0
+datasets_to_include = HCP
+decoder_embed_dim = 512
+grad_accumulation_steps = 1
+grad_clip = 1.0
+gsr = False
+hcp_flat_path = /weka/proj-medarc/shared/HCP-Flat
+mask_ratio = 0.75
+model_name = HCPflat_large_gsrFalse_
+no_qkv_bias = False
+norm_pix_loss = False
+nsd_flat_path = /weka/proj-medarc/shared/NSD-Flat
+num_epochs = 100
+num_frames = 16
+num_samples_per_epoch = 200000
+num_workers = 10
+patch_size = 16
+pct_masks_to_decode = 1
+plotting = True
+pred_t_dim = 8
+print_interval = 20
+probe_base_lr = 0.0003
+probe_batch_size = 8
+probe_num_epochs = 30
+probe_num_samples_per_epoch = 100000
+resume_from_ckpt = True
+seed = 42
+sep_pos_embed = True
+t_patch_size = 2
+test_num_samples_per_epoch = 50000
+test_set = False
+trunc_init = False
+use_contrastive_loss = False
+wandb_log = True
+
+
+WORLD_SIZE=1
+PID of this process = 2177170
+global_pool = True
+gsr = False
+Creating datasets
+Datasets ready
+img_size (144, 320) patch_size (16, 16) frames 16 t_patch_size 2
+model initialized
+latest_checkpoint: epoch99.pth
+
+Loaded checkpoint epoch99.pth from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_
+
+Input dimension: 1024
+total_steps 139140
+wandb_config:
+ {'model_name': 'HCPflat_large_gsrFalse__HCP_FT_age', 'batch_size': 16, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42, 'lr_scheduler_type': 'cycle', 'save_ckpt': False, 'max_lr': 0.0001, 'target': 'age', 'num_workers': 10}
+wandb_id: HCPflat_large_gsrFalse__beta_age_HCPFT_768d7c19-0891-45ae-8cff-9b10e484c8fe

541358.err

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2204713a23f58ebafbeda425a77ff1a4e7bea60e5b934e8fb8e8e1e655c0b9b
+size 13423280

541358.out

@@ -0,0 +1,1470 @@
+NUM_GPUS=1
+MASTER_ADDR=ip-10-0-135-126
+MASTER_PORT=11775
+WORLD_SIZE=1
+------ ARGS ------- 
+ Namespace(found_model_name='HCPflat_large_gsrFalse_', epoch_checkpoint='epoch99.pth', model_suffix='beta', hcp_flat_path='/weka/proj-medarc/shared/HCP-Flat', batch_size=16, wandb_log=True, num_epochs=20, lr_scheduler_type='cycle', save_ckpt=False, seed=42, max_lr=0.0001, target='age', num_workers=10, weight_decay=1e-05, global_pool=True)
+outdir /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_
+Loaded config.yaml from ckpt folder /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_
+
+__CONFIG__
+base_lr = 0.001
+batch_size = 32
+ckpt_interval = 5
+ckpt_saving = True
+cls_embed = True
+contrastive_loss_weight = 1.0
+datasets_to_include = HCP
+decoder_embed_dim = 512
+grad_accumulation_steps = 1
+grad_clip = 1.0
+gsr = False
+hcp_flat_path = /weka/proj-medarc/shared/HCP-Flat
+mask_ratio = 0.75
+model_name = HCPflat_large_gsrFalse_
+no_qkv_bias = False
+norm_pix_loss = False
+nsd_flat_path = /weka/proj-medarc/shared/NSD-Flat
+num_epochs = 100
+num_frames = 16
+num_samples_per_epoch = 200000
+num_workers = 10
+patch_size = 16
+pct_masks_to_decode = 1
+plotting = True
+pred_t_dim = 8
+print_interval = 20
+probe_base_lr = 0.0003
+probe_batch_size = 8
+probe_num_epochs = 30
+probe_num_samples_per_epoch = 100000
+resume_from_ckpt = True
+seed = 42
+sep_pos_embed = True
+t_patch_size = 2
+test_num_samples_per_epoch = 50000
+test_set = False
+trunc_init = False
+use_contrastive_loss = False
+wandb_log = True
+
+
+WORLD_SIZE=1
+PID of this process = 2179123
+global_pool = True
+gsr = False
+Creating datasets
+Datasets ready
+img_size (144, 320) patch_size (16, 16) frames 16 t_patch_size 2
+model initialized
+latest_checkpoint: epoch99.pth
+
+Loaded checkpoint epoch99.pth from /weka/proj-fmri/ckadirt/fMRI-foundation-model/src/checkpoints/HCPflat_large_gsrFalse_
+
+Input dimension: 1024
+total_steps 139140
+wandb_config:
+ {'model_name': 'HCPflat_large_gsrFalse__HCP_FT_age', 'batch_size': 16, 'weight_decay': 1e-05, 'num_epochs': 20, 'seed': 42, 'lr_scheduler_type': 'cycle', 'save_ckpt': False, 'max_lr': 0.0001, 'target': 'age', 'num_workers': 10}
+wandb_id: HCPflat_large_gsrFalse__beta_age_HCPFT_d344d64d-8300-4465-8e75-b35200117944
+Step [100/6957] - Training Loss: 0.3353 - Training MSE: 0.0294
+Step [200/6957] - Training Loss: 0.2611 - Training MSE: 0.0268
+Step [300/6957] - Training Loss: 0.4378 - Training MSE: 0.0261
+Step [400/6957] - Training Loss: 0.5356 - Training MSE: 0.0252
+Step [500/6957] - Training Loss: 0.4324 - Training MSE: 0.0248
+Step [600/6957] - Training Loss: 0.4099 - Training MSE: 0.0242
+Step [700/6957] - Training Loss: 0.4536 - Training MSE: 0.0239
+Step [800/6957] - Training Loss: 0.4616 - Training MSE: 0.0238
+Step [900/6957] - Training Loss: 0.5061 - Training MSE: 0.0237
+Step [1000/6957] - Training Loss: 0.3758 - Training MSE: 0.0235
+Step [1100/6957] - Training Loss: 0.3988 - Training MSE: 0.0234
+Step [1200/6957] - Training Loss: 0.3735 - Training MSE: 0.0233
+Step [1300/6957] - Training Loss: 0.2402 - Training MSE: 0.0234
+Step [1400/6957] - Training Loss: 0.3710 - Training MSE: 0.0235
+Step [1500/6957] - Training Loss: 0.2673 - Training MSE: 0.0234
+Step [1600/6957] - Training Loss: 0.2171 - Training MSE: 0.0232
+Step [1700/6957] - Training Loss: 0.4864 - Training MSE: 0.0231
+Step [1800/6957] - Training Loss: 0.4091 - Training MSE: 0.0230
+Step [1900/6957] - Training Loss: 0.2278 - Training MSE: 0.0230
+Step [2000/6957] - Training Loss: 0.6462 - Training MSE: 0.0230
+Step [2100/6957] - Training Loss: 0.2934 - Training MSE: 0.0229
+Step [2200/6957] - Training Loss: 0.1503 - Training MSE: 0.0229
+Step [2300/6957] - Training Loss: 0.2982 - Training MSE: 0.0228
+Step [2400/6957] - Training Loss: 0.5785 - Training MSE: 0.0228
+Step [2500/6957] - Training Loss: 0.4260 - Training MSE: 0.0227
+Step [2600/6957] - Training Loss: 0.3524 - Training MSE: 0.0227
+Step [2700/6957] - Training Loss: 0.4194 - Training MSE: 0.0226
+Step [2800/6957] - Training Loss: 0.3239 - Training MSE: 0.0225
+Step [2900/6957] - Training Loss: 0.6227 - Training MSE: 0.0225
+Step [3000/6957] - Training Loss: 0.4483 - Training MSE: 0.0224
+Step [3100/6957] - Training Loss: 0.2457 - Training MSE: 0.0224
+Step [3200/6957] - Training Loss: 0.3748 - Training MSE: 0.0223
+Step [3300/6957] - Training Loss: 0.3349 - Training MSE: 0.0222
+Step [3400/6957] - Training Loss: 0.3584 - Training MSE: 0.0222
+Step [3500/6957] - Training Loss: 0.4761 - Training MSE: 0.0222
+Step [3600/6957] - Training Loss: 0.2872 - Training MSE: 0.0221
+Step [3700/6957] - Training Loss: 0.3720 - Training MSE: 0.0220
+Step [3800/6957] - Training Loss: 0.1850 - Training MSE: 0.0220
+Step [3900/6957] - Training Loss: 0.0889 - Training MSE: 0.0219
+Step [4000/6957] - Training Loss: 0.3948 - Training MSE: 0.0219
+Step [4100/6957] - Training Loss: 0.2272 - Training MSE: 0.0218
+Step [4200/6957] - Training Loss: 0.1170 - Training MSE: 0.0218
+Step [4300/6957] - Training Loss: 0.4015 - Training MSE: 0.0217
+Step [4400/6957] - Training Loss: 0.4192 - Training MSE: 0.0217
+Step [4500/6957] - Training Loss: 0.3057 - Training MSE: 0.0216
+Step [4600/6957] - Training Loss: 0.3710 - Training MSE: 0.0216
+Step [4700/6957] - Training Loss: 0.1154 - Training MSE: 0.0215
+Step [4800/6957] - Training Loss: 0.2176 - Training MSE: 0.0215
+Step [4900/6957] - Training Loss: 0.2078 - Training MSE: 0.0214
+Step [5000/6957] - Training Loss: 0.3973 - Training MSE: 0.0214
+Step [5100/6957] - Training Loss: 0.0728 - Training MSE: 0.0213
+Step [5200/6957] - Training Loss: 0.2381 - Training MSE: 0.0212
+Step [5300/6957] - Training Loss: 0.3896 - Training MSE: 0.0212
+Step [5400/6957] - Training Loss: 0.2126 - Training MSE: 0.0211
+Step [5500/6957] - Training Loss: 0.4339 - Training MSE: 0.0210
+Step [5600/6957] - Training Loss: 0.2140 - Training MSE: 0.0209
+Step [5700/6957] - Training Loss: 0.3126 - Training MSE: 0.0209
+Step [5800/6957] - Training Loss: 0.1269 - Training MSE: 0.0208
+Step [5900/6957] - Training Loss: 0.1721 - Training MSE: 0.0207
+Step [6000/6957] - Training Loss: 0.1690 - Training MSE: 0.0206
+Step [6100/6957] - Training Loss: 0.0747 - Training MSE: 0.0205
+Step [6200/6957] - Training Loss: 0.3791 - Training MSE: 0.0204
+Step [6300/6957] - Training Loss: 0.3069 - Training MSE: 0.0203
+Step [6400/6957] - Training Loss: 0.1519 - Training MSE: 0.0203
+Step [6500/6957] - Training Loss: 0.3105 - Training MSE: 0.0202
+Step [6600/6957] - Training Loss: 0.2934 - Training MSE: 0.0201
+Step [6700/6957] - Training Loss: 0.2333 - Training MSE: 0.0200
+Step [6800/6957] - Training Loss: 0.1637 - Training MSE: 0.0199
+Step [6900/6957] - Training Loss: 0.3849 - Training MSE: 0.0198
+Epoch [1/20] - Training Loss: 0.3168, Training MSE: 0.0198 - Validation Loss: 0.3809, Validation MSE: 0.0238
+Step [100/6957] - Training Loss: 0.2016 - Training MSE: 0.0111
+Step [200/6957] - Training Loss: 0.0951 - Training MSE: 0.0108
+Step [300/6957] - Training Loss: 0.0852 - Training MSE: 0.0104
+Step [400/6957] - Training Loss: 0.1962 - Training MSE: 0.0103
+Step [500/6957] - Training Loss: 0.1836 - Training MSE: 0.0104
+Step [600/6957] - Training Loss: 0.1019 - Training MSE: 0.0104
+Step [700/6957] - Training Loss: 0.1209 - Training MSE: 0.0104
+Step [800/6957] - Training Loss: 0.1136 - Training MSE: 0.0103
+Step [900/6957] - Training Loss: 0.1949 - Training MSE: 0.0106
+Step [1000/6957] - Training Loss: 0.0763 - Training MSE: 0.0104
+Step [1100/6957] - Training Loss: 0.1061 - Training MSE: 0.0103
+Step [1200/6957] - Training Loss: 0.2156 - Training MSE: 0.0103
+Step [1300/6957] - Training Loss: 0.1115 - Training MSE: 0.0102
+Step [1400/6957] - Training Loss: 0.1409 - Training MSE: 0.0102
+Step [1500/6957] - Training Loss: 0.1089 - Training MSE: 0.0102
+Step [1600/6957] - Training Loss: 0.1377 - Training MSE: 0.0101
+Step [1700/6957] - Training Loss: 0.1520 - Training MSE: 0.0101
+Step [1800/6957] - Training Loss: 0.1482 - Training MSE: 0.0100
+Step [1900/6957] - Training Loss: 0.1291 - Training MSE: 0.0099
+Step [2000/6957] - Training Loss: 0.0612 - Training MSE: 0.0098
+Step [2100/6957] - Training Loss: 0.1121 - Training MSE: 0.0098
+Step [2200/6957] - Training Loss: 0.0997 - Training MSE: 0.0097
+Step [2300/6957] - Training Loss: 0.1223 - Training MSE: 0.0096
+Step [2400/6957] - Training Loss: 0.1095 - Training MSE: 0.0095
+Step [2500/6957] - Training Loss: 0.2307 - Training MSE: 0.0094
+Step [2600/6957] - Training Loss: 0.0904 - Training MSE: 0.0093
+Step [2700/6957] - Training Loss: 0.0651 - Training MSE: 0.0093
+Step [2800/6957] - Training Loss: 0.0555 - Training MSE: 0.0092
+Step [2900/6957] - Training Loss: 0.0777 - Training MSE: 0.0091
+Step [3000/6957] - Training Loss: 0.1194 - Training MSE: 0.0091
+Step [3100/6957] - Training Loss: 0.1656 - Training MSE: 0.0090
+Step [3200/6957] - Training Loss: 0.0290 - Training MSE: 0.0089
+Step [3300/6957] - Training Loss: 0.0567 - Training MSE: 0.0088
+Step [3400/6957] - Training Loss: 0.1812 - Training MSE: 0.0088
+Step [3500/6957] - Training Loss: 0.0548 - Training MSE: 0.0088
+Step [3600/6957] - Training Loss: 0.0523 - Training MSE: 0.0087
+Step [3700/6957] - Training Loss: 0.0279 - Training MSE: 0.0087
+Step [3800/6957] - Training Loss: 0.1982 - Training MSE: 0.0086
+Step [3900/6957] - Training Loss: 0.0840 - Training MSE: 0.0086
+Step [4000/6957] - Training Loss: 0.0642 - Training MSE: 0.0085
+Step [4100/6957] - Training Loss: 0.1927 - Training MSE: 0.0084
+Step [4200/6957] - Training Loss: 0.0618 - Training MSE: 0.0084
+Step [4300/6957] - Training Loss: 0.1261 - Training MSE: 0.0083
+Step [4400/6957] - Training Loss: 0.0420 - Training MSE: 0.0082
+Step [4500/6957] - Training Loss: 0.0786 - Training MSE: 0.0082
+Step [4600/6957] - Training Loss: 0.0549 - Training MSE: 0.0081
+Step [4700/6957] - Training Loss: 0.0352 - Training MSE: 0.0080
+Step [4800/6957] - Training Loss: 0.0486 - Training MSE: 0.0079
+Step [4900/6957] - Training Loss: 0.0882 - Training MSE: 0.0079
+Step [5000/6957] - Training Loss: 0.0627 - Training MSE: 0.0078
+Step [5100/6957] - Training Loss: 0.1265 - Training MSE: 0.0078
+Step [5200/6957] - Training Loss: 0.0153 - Training MSE: 0.0077
+Step [5300/6957] - Training Loss: 0.0762 - Training MSE: 0.0077
+Step [5400/6957] - Training Loss: 0.0535 - Training MSE: 0.0076
+Step [5500/6957] - Training Loss: 0.0420 - Training MSE: 0.0076
+Step [5600/6957] - Training Loss: 0.0499 - Training MSE: 0.0076
+Step [5700/6957] - Training Loss: 0.0614 - Training MSE: 0.0075
+Step [5800/6957] - Training Loss: 0.1466 - Training MSE: 0.0075
+Step [5900/6957] - Training Loss: 0.0475 - Training MSE: 0.0075
+Step [6000/6957] - Training Loss: 0.1005 - Training MSE: 0.0074
+Step [6100/6957] - Training Loss: 0.0621 - Training MSE: 0.0073
+Step [6200/6957] - Training Loss: 0.0626 - Training MSE: 0.0073
+Step [6300/6957] - Training Loss: 0.1202 - Training MSE: 0.0072
+Step [6400/6957] - Training Loss: 0.0299 - Training MSE: 0.0072
+Step [6500/6957] - Training Loss: 0.0393 - Training MSE: 0.0071
+Step [6600/6957] - Training Loss: 0.0790 - Training MSE: 0.0071
+Step [6700/6957] - Training Loss: 0.1032 - Training MSE: 0.0071
+Step [6800/6957] - Training Loss: 0.0074 - Training MSE: 0.0070
+Step [6900/6957] - Training Loss: 0.0399 - Training MSE: 0.0070
+Epoch [2/20] - Training Loss: 0.1113, Training MSE: 0.0070 - Validation Loss: 0.3821, Validation MSE: 0.0239
+Step [100/6957] - Training Loss: 0.0182 - Training MSE: 0.0025
+Step [200/6957] - Training Loss: 0.0437 - Training MSE: 0.0028
+Step [300/6957] - Training Loss: 0.0455 - Training MSE: 0.0028
+Step [400/6957] - Training Loss: 0.0232 - Training MSE: 0.0027
+Step [500/6957] - Training Loss: 0.0341 - Training MSE: 0.0027
+Step [600/6957] - Training Loss: 0.0408 - Training MSE: 0.0028
+Step [700/6957] - Training Loss: 0.0904 - Training MSE: 0.0031
+Step [800/6957] - Training Loss: 0.0435 - Training MSE: 0.0034
+Step [900/6957] - Training Loss: 0.0163 - Training MSE: 0.0033
+Step [1000/6957] - Training Loss: 0.0254 - Training MSE: 0.0033
+Step [1100/6957] - Training Loss: 0.0347 - Training MSE: 0.0033
+Step [1200/6957] - Training Loss: 0.1033 - Training MSE: 0.0033
+Step [1300/6957] - Training Loss: 0.0276 - Training MSE: 0.0033
+Step [1400/6957] - Training Loss: 0.0292 - Training MSE: 0.0033
+Step [1500/6957] - Training Loss: 0.0164 - Training MSE: 0.0033
+Step [1600/6957] - Training Loss: 0.0184 - Training MSE: 0.0032
+Step [1700/6957] - Training Loss: 0.0209 - Training MSE: 0.0032
+Step [1800/6957] - Training Loss: 0.0255 - Training MSE: 0.0031
+Step [1900/6957] - Training Loss: 0.0433 - Training MSE: 0.0031
+Step [2000/6957] - Training Loss: 0.0994 - Training MSE: 0.0031
+Step [2100/6957] - Training Loss: 0.0184 - Training MSE: 0.0031
+Step [2200/6957] - Training Loss: 0.0773 - Training MSE: 0.0031
+Step [2300/6957] - Training Loss: 0.0191 - Training MSE: 0.0031
+Step [2400/6957] - Training Loss: 0.0062 - Training MSE: 0.0030
+Step [2500/6957] - Training Loss: 0.0859 - Training MSE: 0.0030
+Step [2600/6957] - Training Loss: 0.0404 - Training MSE: 0.0030
+Step [2700/6957] - Training Loss: 0.1087 - Training MSE: 0.0031
+Step [2800/6957] - Training Loss: 0.0550 - Training MSE: 0.0031
+Step [2900/6957] - Training Loss: 0.0737 - Training MSE: 0.0031
+Step [3000/6957] - Training Loss: 0.0278 - Training MSE: 0.0031
+Step [3100/6957] - Training Loss: 0.0095 - Training MSE: 0.0030
+Step [3200/6957] - Training Loss: 0.0365 - Training MSE: 0.0030
+Step [3300/6957] - Training Loss: 0.0267 - Training MSE: 0.0030
+Step [3400/6957] - Training Loss: 0.0054 - Training MSE: 0.0029
+Step [3500/6957] - Training Loss: 0.0677 - Training MSE: 0.0029
+Step [3600/6957] - Training Loss: 0.0237 - Training MSE: 0.0029
+Step [3700/6957] - Training Loss: 0.0996 - Training MSE: 0.0029
+Step [3800/6957] - Training Loss: 0.0062 - Training MSE: 0.0029
+Step [3900/6957] - Training Loss: 0.1073 - Training MSE: 0.0029
+Step [4000/6957] - Training Loss: 0.0340 - Training MSE: 0.0029
+Step [4100/6957] - Training Loss: 0.0457 - Training MSE: 0.0028
+Step [4200/6957] - Training Loss: 0.0783 - Training MSE: 0.0028
+Step [4300/6957] - Training Loss: 0.0980 - Training MSE: 0.0028
+Step [4400/6957] - Training Loss: 0.0407 - Training MSE: 0.0028
+Step [4500/6957] - Training Loss: 0.1220 - Training MSE: 0.0028
+Step [4600/6957] - Training Loss: 0.0222 - Training MSE: 0.0028
+Step [4700/6957] - Training Loss: 0.0079 - Training MSE: 0.0028
+Step [4800/6957] - Training Loss: 0.0223 - Training MSE: 0.0027
+Step [4900/6957] - Training Loss: 0.0140 - Training MSE: 0.0027
+Step [5000/6957] - Training Loss: 0.0385 - Training MSE: 0.0027
+Step [5100/6957] - Training Loss: 0.1232 - Training MSE: 0.0027
+Step [5200/6957] - Training Loss: 0.0204 - Training MSE: 0.0027
+Step [5300/6957] - Training Loss: 0.0187 - Training MSE: 0.0027
+Step [5400/6957] - Training Loss: 0.0767 - Training MSE: 0.0027
+Step [5500/6957] - Training Loss: 0.0465 - Training MSE: 0.0027
+Step [5600/6957] - Training Loss: 0.0126 - Training MSE: 0.0027
+Step [5700/6957] - Training Loss: 0.0061 - Training MSE: 0.0027
+Step [5800/6957] - Training Loss: 0.0120 - Training MSE: 0.0027
+Step [5900/6957] - Training Loss: 0.0580 - Training MSE: 0.0027
+Step [6000/6957] - Training Loss: 0.0131 - Training MSE: 0.0027
+Step [6100/6957] - Training Loss: 0.0497 - Training MSE: 0.0027
+Step [6200/6957] - Training Loss: 0.0713 - Training MSE: 0.0027
+Step [6300/6957] - Training Loss: 0.0221 - Training MSE: 0.0027
+Step [6400/6957] - Training Loss: 0.0835 - Training MSE: 0.0027
+Step [6500/6957] - Training Loss: 0.0409 - Training MSE: 0.0027
+Step [6600/6957] - Training Loss: 0.0232 - Training MSE: 0.0027
+Step [6700/6957] - Training Loss: 0.0029 - Training MSE: 0.0026
+Step [6800/6957] - Training Loss: 0.0060 - Training MSE: 0.0026
+Step [6900/6957] - Training Loss: 0.0543 - Training MSE: 0.0026
+Epoch [3/20] - Training Loss: 0.0415, Training MSE: 0.0026 - Validation Loss: 0.4569, Validation MSE: 0.0286
+Step [100/6957] - Training Loss: 0.1452 - Training MSE: 0.0022
+Step [200/6957] - Training Loss: 0.0052 - Training MSE: 0.0024
+Step [300/6957] - Training Loss: 0.0096 - Training MSE: 0.0021
+Step [400/6957] - Training Loss: 0.0192 - Training MSE: 0.0023
+Step [500/6957] - Training Loss: 0.0191 - Training MSE: 0.0021
+Step [600/6957] - Training Loss: 0.0040 - Training MSE: 0.0018
+Step [700/6957] - Training Loss: 0.0555 - Training MSE: 0.0018
+Step [800/6957] - Training Loss: 0.0046 - Training MSE: 0.0017
+Step [900/6957] - Training Loss: 0.0146 - Training MSE: 0.0017
+Step [1000/6957] - Training Loss: 0.0029 - Training MSE: 0.0016
+Step [1100/6957] - Training Loss: 0.0076 - Training MSE: 0.0016
+Step [1200/6957] - Training Loss: 0.0017 - Training MSE: 0.0016
+Step [1300/6957] - Training Loss: 0.2443 - Training MSE: 0.0016
+Step [1400/6957] - Training Loss: 0.0048 - Training MSE: 0.0016
+Step [1500/6957] - Training Loss: 0.0084 - Training MSE: 0.0015
+Step [1600/6957] - Training Loss: 0.0072 - Training MSE: 0.0015
+Step [1700/6957] - Training Loss: 0.0022 - Training MSE: 0.0014
+Step [1800/6957] - Training Loss: 0.0091 - Training MSE: 0.0014
+Step [1900/6957] - Training Loss: 0.0623 - Training MSE: 0.0015
+Step [2000/6957] - Training Loss: 0.0351 - Training MSE: 0.0015
+Step [2100/6957] - Training Loss: 0.0310 - Training MSE: 0.0015
+Step [2200/6957] - Training Loss: 0.0043 - Training MSE: 0.0015
+Step [2300/6957] - Training Loss: 0.1778 - Training MSE: 0.0015
+Step [2400/6957] - Training Loss: 0.0053 - Training MSE: 0.0015
+Step [2500/6957] - Training Loss: 0.0027 - Training MSE: 0.0015
+Step [2600/6957] - Training Loss: 0.0096 - Training MSE: 0.0015
+Step [2700/6957] - Training Loss: 0.0045 - Training MSE: 0.0015
+Step [2800/6957] - Training Loss: 0.0222 - Training MSE: 0.0016
+Step [2900/6957] - Training Loss: 0.0025 - Training MSE: 0.0016
+Step [3000/6957] - Training Loss: 0.0044 - Training MSE: 0.0016
+Step [3100/6957] - Training Loss: 0.0071 - Training MSE: 0.0015
+Step [3200/6957] - Training Loss: 0.0092 - Training MSE: 0.0015
+Step [3300/6957] - Training Loss: 0.0054 - Training MSE: 0.0015
+Step [3400/6957] - Training Loss: 0.0041 - Training MSE: 0.0015
+Step [3500/6957] - Training Loss: 0.0016 - Training MSE: 0.0015
+Step [3600/6957] - Training Loss: 0.0061 - Training MSE: 0.0015
+Step [3700/6957] - Training Loss: 0.0033 - Training MSE: 0.0015
+Step [3800/6957] - Training Loss: 0.0051 - Training MSE: 0.0015
+Step [3900/6957] - Training Loss: 0.0161 - Training MSE: 0.0015
+Step [4000/6957] - Training Loss: 0.0048 - Training MSE: 0.0015
+Step [4100/6957] - Training Loss: 0.0014 - Training MSE: 0.0015
+Step [4200/6957] - Training Loss: 0.0726 - Training MSE: 0.0015
+Step [4300/6957] - Training Loss: 0.0725 - Training MSE: 0.0015
+Step [4400/6957] - Training Loss: 0.0185 - Training MSE: 0.0015
+Step [4500/6957] - Training Loss: 0.0060 - Training MSE: 0.0015
+Step [4600/6957] - Training Loss: 0.0278 - Training MSE: 0.0015
+Step [4700/6957] - Training Loss: 0.0419 - Training MSE: 0.0014
+Step [4800/6957] - Training Loss: 0.0030 - Training MSE: 0.0014
+Step [4900/6957] - Training Loss: 0.1963 - Training MSE: 0.0015
+Step [5000/6957] - Training Loss: 0.0031 - Training MSE: 0.0015
+Step [5100/6957] - Training Loss: 0.0334 - Training MSE: 0.0015
+Step [5200/6957] - Training Loss: 0.0034 - Training MSE: 0.0015
+Step [5300/6957] - Training Loss: 0.0039 - Training MSE: 0.0015
+Step [5400/6957] - Training Loss: 0.0140 - Training MSE: 0.0015
+Step [5500/6957] - Training Loss: 0.0021 - Training MSE: 0.0015
+Step [5600/6957] - Training Loss: 0.0461 - Training MSE: 0.0015
+Step [5700/6957] - Training Loss: 0.0448 - Training MSE: 0.0015
+Step [5800/6957] - Training Loss: 0.0549 - Training MSE: 0.0015
+Step [5900/6957] - Training Loss: 0.0014 - Training MSE: 0.0015
+Step [6000/6957] - Training Loss: 0.0355 - Training MSE: 0.0015
+Step [6100/6957] - Training Loss: 0.0026 - Training MSE: 0.0015
+Step [6200/6957] - Training Loss: 0.0021 - Training MSE: 0.0015
+Step [6300/6957] - Training Loss: 0.0119 - Training MSE: 0.0015
+Step [6400/6957] - Training Loss: 0.0017 - Training MSE: 0.0015
+Step [6500/6957] - Training Loss: 0.0041 - Training MSE: 0.0015
+Step [6600/6957] - Training Loss: 0.0021 - Training MSE: 0.0015
+Step [6700/6957] - Training Loss: 0.0290 - Training MSE: 0.0014
+Step [6800/6957] - Training Loss: 0.0036 - Training MSE: 0.0014
+Step [6900/6957] - Training Loss: 0.0089 - Training MSE: 0.0015
+Epoch [4/20] - Training Loss: 0.0233, Training MSE: 0.0015 - Validation Loss: 0.4694, Validation MSE: 0.0293
+Step [100/6957] - Training Loss: 0.0033 - Training MSE: 0.0014
+Step [200/6957] - Training Loss: 0.0049 - Training MSE: 0.0014
+Step [300/6957] - Training Loss: 0.0239 - Training MSE: 0.0014
+Step [400/6957] - Training Loss: 0.0086 - Training MSE: 0.0012
+Step [500/6957] - Training Loss: 0.0233 - Training MSE: 0.0010
+Step [600/6957] - Training Loss: 0.0147 - Training MSE: 0.0010
+Step [700/6957] - Training Loss: 0.0022 - Training MSE: 0.0009
+Step [800/6957] - Training Loss: 0.0691 - Training MSE: 0.0010
+Step [900/6957] - Training Loss: 0.0010 - Training MSE: 0.0010
+Step [1000/6957] - Training Loss: 0.0088 - Training MSE: 0.0011
+Step [1100/6957] - Training Loss: 0.0432 - Training MSE: 0.0010
+Step [1200/6957] - Training Loss: 0.0146 - Training MSE: 0.0010
+Step [1300/6957] - Training Loss: 0.0025 - Training MSE: 0.0010
+Step [1400/6957] - Training Loss: 0.0050 - Training MSE: 0.0010
+Step [1500/6957] - Training Loss: 0.0044 - Training MSE: 0.0011
+Step [1600/6957] - Training Loss: 0.0100 - Training MSE: 0.0011
+Step [1700/6957] - Training Loss: 0.0373 - Training MSE: 0.0011
+Step [1800/6957] - Training Loss: 0.0562 - Training MSE: 0.0011
+Step [1900/6957] - Training Loss: 0.0343 - Training MSE: 0.0012
+Step [2000/6957] - Training Loss: 0.0393 - Training MSE: 0.0012
+Step [2100/6957] - Training Loss: 0.0077 - Training MSE: 0.0012
+Step [2200/6957] - Training Loss: 0.0811 - Training MSE: 0.0012
+Step [2300/6957] - Training Loss: 0.0287 - Training MSE: 0.0012
+Step [2400/6957] - Training Loss: 0.0049 - Training MSE: 0.0012
+Step [2500/6957] - Training Loss: 0.0102 - Training MSE: 0.0012
+Step [2600/6957] - Training Loss: 0.0524 - Training MSE: 0.0012
+Step [2700/6957] - Training Loss: 0.0213 - Training MSE: 0.0012
+Step [2800/6957] - Training Loss: 0.0011 - Training MSE: 0.0012
+Step [2900/6957] - Training Loss: 0.0007 - Training MSE: 0.0012
+Step [3000/6957] - Training Loss: 0.0252 - Training MSE: 0.0012
+Step [3100/6957] - Training Loss: 0.0061 - Training MSE: 0.0012
+Step [3200/6957] - Training Loss: 0.0013 - Training MSE: 0.0012
+Step [3300/6957] - Training Loss: 0.0046 - Training MSE: 0.0012
+Step [3400/6957] - Training Loss: 0.0386 - Training MSE: 0.0012
+Step [3500/6957] - Training Loss: 0.0021 - Training MSE: 0.0012
+Step [3600/6957] - Training Loss: 0.0023 - Training MSE: 0.0012
+Step [3700/6957] - Training Loss: 0.0050 - Training MSE: 0.0011
+Step [3800/6957] - Training Loss: 0.0320 - Training MSE: 0.0011
+Step [3900/6957] - Training Loss: 0.0047 - Training MSE: 0.0011
+Step [4000/6957] - Training Loss: 0.0053 - Training MSE: 0.0011
+Step [4100/6957] - Training Loss: 0.0029 - Training MSE: 0.0011
+Step [4200/6957] - Training Loss: 0.0021 - Training MSE: 0.0011
+Step [4300/6957] - Training Loss: 0.0082 - Training MSE: 0.0011
+Step [4400/6957] - Training Loss: 0.0021 - Training MSE: 0.0011
+Step [4500/6957] - Training Loss: 0.0034 - Training MSE: 0.0011
+Step [4600/6957] - Training Loss: 0.0016 - Training MSE: 0.0011
+Step [4700/6957] - Training Loss: 0.0408 - Training MSE: 0.0012
+Step [4800/6957] - Training Loss: 0.0056 - Training MSE: 0.0012
+Step [4900/6957] - Training Loss: 0.0018 - Training MSE: 0.0012
+Step [5000/6957] - Training Loss: 0.0031 - Training MSE: 0.0011
+Step [5100/6957] - Training Loss: 0.0463 - Training MSE: 0.0011
+Step [5200/6957] - Training Loss: 0.0009 - Training MSE: 0.0011
+Step [5300/6957] - Training Loss: 0.0042 - Training MSE: 0.0011
+Step [5400/6957] - Training Loss: 0.0024 - Training MSE: 0.0011
+Step [5500/6957] - Training Loss: 0.0071 - Training MSE: 0.0011
+Step [5600/6957] - Training Loss: 0.0008 - Training MSE: 0.0011
+Step [5700/6957] - Training Loss: 0.0438 - Training MSE: 0.0011
+Step [5800/6957] - Training Loss: 0.0116 - Training MSE: 0.0011
+Step [5900/6957] - Training Loss: 0.0138 - Training MSE: 0.0011
+Step [6000/6957] - Training Loss: 0.0010 - Training MSE: 0.0011
+Step [6100/6957] - Training Loss: 0.0008 - Training MSE: 0.0011
+Step [6200/6957] - Training Loss: 0.0719 - Training MSE: 0.0011
+Step [6300/6957] - Training Loss: 0.0007 - Training MSE: 0.0011
+Step [6400/6957] - Training Loss: 0.0035 - Training MSE: 0.0011
+Step [6500/6957] - Training Loss: 0.0058 - Training MSE: 0.0011
+Step [6600/6957] - Training Loss: 0.0007 - Training MSE: 0.0011
+Step [6700/6957] - Training Loss: 0.0015 - Training MSE: 0.0011
+Step [6800/6957] - Training Loss: 0.0011 - Training MSE: 0.0011
+Step [6900/6957] - Training Loss: 0.0561 - Training MSE: 0.0011
+Epoch [5/20] - Training Loss: 0.0174, Training MSE: 0.0011 - Validation Loss: 0.4568, Validation MSE: 0.0285
+Step [100/6957] - Training Loss: 0.0019 - Training MSE: 0.0006
+Step [200/6957] - Training Loss: 0.0025 - Training MSE: 0.0006
+Step [300/6957] - Training Loss: 0.0003 - Training MSE: 0.0006
+Step [400/6957] - Training Loss: 0.0070 - Training MSE: 0.0006
+Step [500/6957] - Training Loss: 0.0369 - Training MSE: 0.0006
+Step [600/6957] - Training Loss: 0.0011 - Training MSE: 0.0006
+Step [700/6957] - Training Loss: 0.0022 - Training MSE: 0.0007
+Step [800/6957] - Training Loss: 0.0053 - Training MSE: 0.0007
+Step [900/6957] - Training Loss: 0.0101 - Training MSE: 0.0008
+Step [1000/6957] - Training Loss: 0.0458 - Training MSE: 0.0008
+Step [1100/6957] - Training Loss: 0.0078 - Training MSE: 0.0008
+Step [1200/6957] - Training Loss: 0.0010 - Training MSE: 0.0008
+Step [1300/6957] - Training Loss: 0.0014 - Training MSE: 0.0008
+Step [1400/6957] - Training Loss: 0.0018 - Training MSE: 0.0008
+Step [1500/6957] - Training Loss: 0.0660 - Training MSE: 0.0008
+Step [1600/6957] - Training Loss: 0.0113 - Training MSE: 0.0009
+Step [1700/6957] - Training Loss: 0.0038 - Training MSE: 0.0010
+Step [1800/6957] - Training Loss: 0.0625 - Training MSE: 0.0010
+Step [1900/6957] - Training Loss: 0.0008 - Training MSE: 0.0010
+Step [2000/6957] - Training Loss: 0.0186 - Training MSE: 0.0011
+Step [2100/6957] - Training Loss: 0.0016 - Training MSE: 0.0010
+Step [2200/6957] - Training Loss: 0.0019 - Training MSE: 0.0010
+Step [2300/6957] - Training Loss: 0.0005 - Training MSE: 0.0010
+Step [2400/6957] - Training Loss: 0.0057 - Training MSE: 0.0010
+Step [2500/6957] - Training Loss: 0.0435 - Training MSE: 0.0010
+Step [2600/6957] - Training Loss: 0.0101 - Training MSE: 0.0010
+Step [2700/6957] - Training Loss: 0.0023 - Training MSE: 0.0010
+Step [2800/6957] - Training Loss: 0.0014 - Training MSE: 0.0010
+Step [2900/6957] - Training Loss: 0.0008 - Training MSE: 0.0010
+Step [3000/6957] - Training Loss: 0.0002 - Training MSE: 0.0010
+Step [3100/6957] - Training Loss: 0.0017 - Training MSE: 0.0009
+Step [3200/6957] - Training Loss: 0.0030 - Training MSE: 0.0009
+Step [3300/6957] - Training Loss: 0.0011 - Training MSE: 0.0009
+Step [3400/6957] - Training Loss: 0.0027 - Training MSE: 0.0009
+Step [3500/6957] - Training Loss: 0.0016 - Training MSE: 0.0009
+Step [3600/6957] - Training Loss: 0.0071 - Training MSE: 0.0009
+Step [3700/6957] - Training Loss: 0.0020 - Training MSE: 0.0009
+Step [3800/6957] - Training Loss: 0.0609 - Training MSE: 0.0009
+Step [3900/6957] - Training Loss: 0.0014 - Training MSE: 0.0009
+Step [4000/6957] - Training Loss: 0.0020 - Training MSE: 0.0009
+Step [4100/6957] - Training Loss: 0.0459 - Training MSE: 0.0009
+Step [4200/6957] - Training Loss: 0.0027 - Training MSE: 0.0009
+Step [4300/6957] - Training Loss: 0.0003 - Training MSE: 0.0009
+Step [4400/6957] - Training Loss: 0.0009 - Training MSE: 0.0009
+Step [4500/6957] - Training Loss: 0.0010 - Training MSE: 0.0008
+Step [4600/6957] - Training Loss: 0.0004 - Training MSE: 0.0008
+Step [4700/6957] - Training Loss: 0.0223 - Training MSE: 0.0008
+Step [4800/6957] - Training Loss: 0.0084 - Training MSE: 0.0009
+Step [4900/6957] - Training Loss: 0.0016 - Training MSE: 0.0009
+Step [5000/6957] - Training Loss: 0.0214 - Training MSE: 0.0009
+Step [5100/6957] - Training Loss: 0.0050 - Training MSE: 0.0009
+Step [5200/6957] - Training Loss: 0.0034 - Training MSE: 0.0009
+Step [5300/6957] - Training Loss: 0.0339 - Training MSE: 0.0009
+Step [5400/6957] - Training Loss: 0.0222 - Training MSE: 0.0009
+Step [5500/6957] - Training Loss: 0.0016 - Training MSE: 0.0009
+Step [5600/6957] - Training Loss: 0.0267 - Training MSE: 0.0009
+Step [5700/6957] - Training Loss: 0.0015 - Training MSE: 0.0009
+Step [5800/6957] - Training Loss: 0.0064 - Training MSE: 0.0009
+Step [5900/6957] - Training Loss: 0.0026 - Training MSE: 0.0008
+Step [6000/6957] - Training Loss: 0.0308 - Training MSE: 0.0008
+Step [6100/6957] - Training Loss: 0.0045 - Training MSE: 0.0008
+Step [6200/6957] - Training Loss: 0.0064 - Training MSE: 0.0008
+Step [6300/6957] - Training Loss: 0.0015 - Training MSE: 0.0008
+Step [6400/6957] - Training Loss: 0.0049 - Training MSE: 0.0008
+Step [6500/6957] - Training Loss: 0.0029 - Training MSE: 0.0008
+Step [6600/6957] - Training Loss: 0.0021 - Training MSE: 0.0008
+Step [6700/6957] - Training Loss: 0.0014 - Training MSE: 0.0008
+Step [6800/6957] - Training Loss: 0.0065 - Training MSE: 0.0008
+Step [6900/6957] - Training Loss: 0.0080 - Training MSE: 0.0008
+Epoch [6/20] - Training Loss: 0.0132, Training MSE: 0.0008 - Validation Loss: 0.4173, Validation MSE: 0.0261
+Step [100/6957] - Training Loss: 0.0090 - Training MSE: 0.0009
+Step [200/6957] - Training Loss: 0.0018 - Training MSE: 0.0007
+Step [300/6957] - Training Loss: 0.0200 - Training MSE: 0.0008
+Step [400/6957] - Training Loss: 0.0077 - Training MSE: 0.0007
+Step [500/6957] - Training Loss: 0.0022 - Training MSE: 0.0008
+Step [600/6957] - Training Loss: 0.0009 - Training MSE: 0.0007
+Step [700/6957] - Training Loss: 0.0011 - Training MSE: 0.0007
+Step [800/6957] - Training Loss: 0.0263 - Training MSE: 0.0007
+Step [900/6957] - Training Loss: 0.1101 - Training MSE: 0.0009
+Step [1000/6957] - Training Loss: 0.0844 - Training MSE: 0.0010
+Step [1100/6957] - Training Loss: 0.0017 - Training MSE: 0.0010
+Step [1200/6957] - Training Loss: 0.0028 - Training MSE: 0.0010
+Step [1300/6957] - Training Loss: 0.0631 - Training MSE: 0.0009
+Step [1400/6957] - Training Loss: 0.0028 - Training MSE: 0.0009
+Step [1500/6957] - Training Loss: 0.0002 - Training MSE: 0.0009
+Step [1600/6957] - Training Loss: 0.0004 - Training MSE: 0.0008
+Step [1700/6957] - Training Loss: 0.0047 - Training MSE: 0.0008
+Step [1800/6957] - Training Loss: 0.0010 - Training MSE: 0.0007
+Step [1900/6957] - Training Loss: 0.0003 - Training MSE: 0.0007
+Step [2000/6957] - Training Loss: 0.0042 - Training MSE: 0.0007
+Step [2100/6957] - Training Loss: 0.0007 - Training MSE: 0.0007
+Step [2200/6957] - Training Loss: 0.0023 - Training MSE: 0.0007
+Step [2300/6957] - Training Loss: 0.0136 - Training MSE: 0.0007
+Step [2400/6957] - Training Loss: 0.0018 - Training MSE: 0.0007
+Step [2500/6957] - Training Loss: 0.0009 - Training MSE: 0.0006
+Step [2600/6957] - Training Loss: 0.0740 - Training MSE: 0.0007
+Step [2700/6957] - Training Loss: 0.0010 - Training MSE: 0.0007
+Step [2800/6957] - Training Loss: 0.0463 - Training MSE: 0.0007
+Step [2900/6957] - Training Loss: 0.0032 - Training MSE: 0.0007
+Step [3000/6957] - Training Loss: 0.0006 - Training MSE: 0.0007
+Step [3100/6957] - Training Loss: 0.0028 - Training MSE: 0.0007
+Step [3200/6957] - Training Loss: 0.0003 - Training MSE: 0.0007
+Step [3300/6957] - Training Loss: 0.0013 - Training MSE: 0.0007
+Step [3400/6957] - Training Loss: 0.0424 - Training MSE: 0.0007
+Step [3500/6957] - Training Loss: 0.0009 - Training MSE: 0.0007
+Step [3600/6957] - Training Loss: 0.0057 - Training MSE: 0.0007
+Step [3700/6957] - Training Loss: 0.0007 - Training MSE: 0.0007
+Step [3800/6957] - Training Loss: 0.0544 - Training MSE: 0.0007
+Step [3900/6957] - Training Loss: 0.0161 - Training MSE: 0.0007
+Step [4000/6957] - Training Loss: 0.0615 - Training MSE: 0.0007
+Step [4100/6957] - Training Loss: 0.0008 - Training MSE: 0.0007
+Step [4200/6957] - Training Loss: 0.0022 - Training MSE: 0.0007
+Step [4300/6957] - Training Loss: 0.0055 - Training MSE: 0.0007
+Step [4400/6957] - Training Loss: 0.0228 - Training MSE: 0.0007
+Step [4500/6957] - Training Loss: 0.0571 - Training MSE: 0.0007
+Step [4600/6957] - Training Loss: 0.0026 - Training MSE: 0.0007
+Step [4700/6957] - Training Loss: 0.0044 - Training MSE: 0.0007
+Step [4800/6957] - Training Loss: 0.0009 - Training MSE: 0.0007
+Step [4900/6957] - Training Loss: 0.0020 - Training MSE: 0.0007
+Step [5000/6957] - Training Loss: 0.0018 - Training MSE: 0.0007
+Step [5100/6957] - Training Loss: 0.0020 - Training MSE: 0.0007
+Step [5200/6957] - Training Loss: 0.0007 - Training MSE: 0.0007
+Step [5300/6957] - Training Loss: 0.0001 - Training MSE: 0.0006
+Step [5400/6957] - Training Loss: 0.0012 - Training MSE: 0.0006
+Step [5500/6957] - Training Loss: 0.0013 - Training MSE: 0.0006
+Step [5600/6957] - Training Loss: 0.0010 - Training MSE: 0.0006
+Step [5700/6957] - Training Loss: 0.0008 - Training MSE: 0.0006
+Step [5800/6957] - Training Loss: 0.0019 - Training MSE: 0.0006
+Step [5900/6957] - Training Loss: 0.0006 - Training MSE: 0.0006
+Step [6000/6957] - Training Loss: 0.0183 - Training MSE: 0.0006
+Step [6100/6957] - Training Loss: 0.0039 - Training MSE: 0.0006
+Step [6200/6957] - Training Loss: 0.0068 - Training MSE: 0.0007
+Step [6300/6957] - Training Loss: 0.0008 - Training MSE: 0.0006
+Step [6400/6957] - Training Loss: 0.0013 - Training MSE: 0.0006
+Step [6500/6957] - Training Loss: 0.0005 - Training MSE: 0.0006
+Step [6600/6957] - Training Loss: 0.0011 - Training MSE: 0.0006
+Step [6700/6957] - Training Loss: 0.0007 - Training MSE: 0.0006
+Step [6800/6957] - Training Loss: 0.0233 - Training MSE: 0.0006
+Step [6900/6957] - Training Loss: 0.0003 - Training MSE: 0.0006
+Epoch [7/20] - Training Loss: 0.0099, Training MSE: 0.0006 - Validation Loss: 0.4181, Validation MSE: 0.0261
+Step [100/6957] - Training Loss: 0.0006 - Training MSE: 0.0001
+Step [200/6957] - Training Loss: 0.0176 - Training MSE: 0.0003
+Step [300/6957] - Training Loss: 0.0064 - Training MSE: 0.0005
+Step [400/6957] - Training Loss: 0.0118 - Training MSE: 0.0007
+Step [500/6957] - Training Loss: 0.0097 - Training MSE: 0.0008
+Step [600/6957] - Training Loss: 0.0232 - Training MSE: 0.0008
+Step [700/6957] - Training Loss: 0.0042 - Training MSE: 0.0008
+Step [800/6957] - Training Loss: 0.0052 - Training MSE: 0.0008
+Step [900/6957] - Training Loss: 0.0006 - Training MSE: 0.0007
+Step [1000/6957] - Training Loss: 0.0091 - Training MSE: 0.0007
+Step [1100/6957] - Training Loss: 0.0007 - Training MSE: 0.0007
+Step [1200/6957] - Training Loss: 0.0022 - Training MSE: 0.0007
+Step [1300/6957] - Training Loss: 0.0080 - Training MSE: 0.0007
+Step [1400/6957] - Training Loss: 0.0019 - Training MSE: 0.0007
+Step [1500/6957] - Training Loss: 0.0040 - Training MSE: 0.0007
+Step [1600/6957] - Training Loss: 0.1144 - Training MSE: 0.0007
+Step [1700/6957] - Training Loss: 0.0018 - Training MSE: 0.0007
+Step [1800/6957] - Training Loss: 0.0035 - Training MSE: 0.0007
+Step [1900/6957] - Training Loss: 0.0012 - Training MSE: 0.0007
+Step [2000/6957] - Training Loss: 0.0017 - Training MSE: 0.0007
+Step [2100/6957] - Training Loss: 0.0026 - Training MSE: 0.0007
+Step [2200/6957] - Training Loss: 0.0009 - Training MSE: 0.0007
+Step [2300/6957] - Training Loss: 0.0008 - Training MSE: 0.0006
+Step [2400/6957] - Training Loss: 0.0004 - Training MSE: 0.0006
+Step [2500/6957] - Training Loss: 0.0014 - Training MSE: 0.0006
+Step [2600/6957] - Training Loss: 0.0021 - Training MSE: 0.0006
+Step [2700/6957] - Training Loss: 0.0010 - Training MSE: 0.0006
+Step [2800/6957] - Training Loss: 0.0189 - Training MSE: 0.0006
+Step [2900/6957] - Training Loss: 0.0054 - Training MSE: 0.0006
+Step [3000/6957] - Training Loss: 0.0007 - Training MSE: 0.0006
+Step [3100/6957] - Training Loss: 0.0018 - Training MSE: 0.0005
+Step [3200/6957] - Training Loss: 0.0009 - Training MSE: 0.0005
+Step [3300/6957] - Training Loss: 0.0005 - Training MSE: 0.0005
+Step [3400/6957] - Training Loss: 0.0006 - Training MSE: 0.0005
+Step [3500/6957] - Training Loss: 0.0288 - Training MSE: 0.0005
+Step [3600/6957] - Training Loss: 0.0011 - Training MSE: 0.0005
+Step [3700/6957] - Training Loss: 0.0032 - Training MSE: 0.0005
+Step [3800/6957] - Training Loss: 0.0194 - Training MSE: 0.0005
+Step [3900/6957] - Training Loss: 0.0020 - Training MSE: 0.0005
+Step [4000/6957] - Training Loss: 0.0042 - Training MSE: 0.0006
+Step [4100/6957] - Training Loss: 0.0014 - Training MSE: 0.0006
+Step [4200/6957] - Training Loss: 0.0225 - Training MSE: 0.0006
+Step [4300/6957] - Training Loss: 0.0025 - Training MSE: 0.0006
+Step [4400/6957] - Training Loss: 0.0004 - Training MSE: 0.0006
+Step [4500/6957] - Training Loss: 0.0003 - Training MSE: 0.0006
+Step [4600/6957] - Training Loss: 0.0008 - Training MSE: 0.0006
+Step [4700/6957] - Training Loss: 0.0004 - Training MSE: 0.0006
+Step [4800/6957] - Training Loss: 0.0012 - Training MSE: 0.0006
+Step [4900/6957] - Training Loss: 0.0006 - Training MSE: 0.0005
+Step [5000/6957] - Training Loss: 0.0028 - Training MSE: 0.0005
+Step [5100/6957] - Training Loss: 0.0076 - Training MSE: 0.0005
+Step [5200/6957] - Training Loss: 0.0012 - Training MSE: 0.0005
+Step [5300/6957] - Training Loss: 0.0009 - Training MSE: 0.0005
+Step [5400/6957] - Training Loss: 0.0023 - Training MSE: 0.0005
+Step [5500/6957] - Training Loss: 0.0082 - Training MSE: 0.0005
+Step [5600/6957] - Training Loss: 0.0004 - Training MSE: 0.0005
+Step [5700/6957] - Training Loss: 0.0004 - Training MSE: 0.0005
+Step [5800/6957] - Training Loss: 0.0015 - Training MSE: 0.0005
+Step [5900/6957] - Training Loss: 0.0004 - Training MSE: 0.0005
+Step [6000/6957] - Training Loss: 0.0010 - Training MSE: 0.0005
+Step [6100/6957] - Training Loss: 0.0036 - Training MSE: 0.0005
+Step [6200/6957] - Training Loss: 0.0045 - Training MSE: 0.0005
+Step [6300/6957] - Training Loss: 0.0175 - Training MSE: 0.0005
+Step [6400/6957] - Training Loss: 0.0007 - Training MSE: 0.0005
+Step [6500/6957] - Training Loss: 0.0359 - Training MSE: 0.0005
+Step [6600/6957] - Training Loss: 0.0230 - Training MSE: 0.0005
+Step [6700/6957] - Training Loss: 0.0022 - Training MSE: 0.0005
+Step [6800/6957] - Training Loss: 0.0004 - Training MSE: 0.0005
+Step [6900/6957] - Training Loss: 0.0012 - Training MSE: 0.0005
+Epoch [8/20] - Training Loss: 0.0084, Training MSE: 0.0005 - Validation Loss: 0.4067, Validation MSE: 0.0254
+Step [100/6957] - Training Loss: 0.0005 - Training MSE: 0.0001
+Step [200/6957] - Training Loss: 0.0091 - Training MSE: 0.0002
+Step [300/6957] - Training Loss: 0.0018 - Training MSE: 0.0002
+Step [400/6957] - Training Loss: 0.0004 - Training MSE: 0.0002
+Step [500/6957] - Training Loss: 0.0002 - Training MSE: 0.0002
+Step [600/6957] - Training Loss: 0.0004 - Training MSE: 0.0001
+Step [700/6957] - Training Loss: 0.0005 - Training MSE: 0.0001
+Step [800/6957] - Training Loss: 0.0003 - Training MSE: 0.0001
+Step [900/6957] - Training Loss: 0.0735 - Training MSE: 0.0002
+Step [1000/6957] - Training Loss: 0.0010 - Training MSE: 0.0003
+Step [1100/6957] - Training Loss: 0.0008 - Training MSE: 0.0003
+Step [1200/6957] - Training Loss: 0.0019 - Training MSE: 0.0003
+Step [1300/6957] - Training Loss: 0.0013 - Training MSE: 0.0003
+Step [1400/6957] - Training Loss: 0.0046 - Training MSE: 0.0003
+Step [1500/6957] - Training Loss: 0.0089 - Training MSE: 0.0004
+Step [1600/6957] - Training Loss: 0.0010 - Training MSE: 0.0004
+Step [1700/6957] - Training Loss: 0.0053 - Training MSE: 0.0004
+Step [1800/6957] - Training Loss: 0.0005 - Training MSE: 0.0004
+Step [1900/6957] - Training Loss: 0.0013 - Training MSE: 0.0004
+Step [2000/6957] - Training Loss: 0.0040 - Training MSE: 0.0004
+Step [2100/6957] - Training Loss: 0.0008 - Training MSE: 0.0003
+Step [2200/6957] - Training Loss: 0.0002 - Training MSE: 0.0003
+Step [2300/6957] - Training Loss: 0.0003 - Training MSE: 0.0003
+Step [2400/6957] - Training Loss: 0.0002 - Training MSE: 0.0003
+Step [2500/6957] - Training Loss: 0.0023 - Training MSE: 0.0003
+Step [2600/6957] - Training Loss: 0.0079 - Training MSE: 0.0004
+Step [2700/6957] - Training Loss: 0.0020 - Training MSE: 0.0004
+Step [2800/6957] - Training Loss: 0.0009 - Training MSE: 0.0004
+Step [2900/6957] - Training Loss: 0.0013 - Training MSE: 0.0004
+Step [3000/6957] - Training Loss: 0.0011 - Training MSE: 0.0004
+Step [3100/6957] - Training Loss: 0.0526 - Training MSE: 0.0004
+Step [3200/6957] - Training Loss: 0.0006 - Training MSE: 0.0004
+Step [3300/6957] - Training Loss: 0.0004 - Training MSE: 0.0004
+Step [3400/6957] - Training Loss: 0.0576 - Training MSE: 0.0004
+Step [3500/6957] - Training Loss: 0.0003 - Training MSE: 0.0004
+Step [3600/6957] - Training Loss: 0.0003 - Training MSE: 0.0004
+Step [3700/6957] - Training Loss: 0.0006 - Training MSE: 0.0004
+Step [3800/6957] - Training Loss: 0.0009 - Training MSE: 0.0004
+Step [3900/6957] - Training Loss: 0.0002 - Training MSE: 0.0004
+Step [4000/6957] - Training Loss: 0.0004 - Training MSE: 0.0004
+Step [4100/6957] - Training Loss: 0.0004 - Training MSE: 0.0004
+Step [4200/6957] - Training Loss: 0.0020 - Training MSE: 0.0004
+Step [4300/6957] - Training Loss: 0.0016 - Training MSE: 0.0004
+Step [4400/6957] - Training Loss: 0.0005 - Training MSE: 0.0004
+Step [4500/6957] - Training Loss: 0.0009 - Training MSE: 0.0004
+Step [4600/6957] - Training Loss: 0.0072 - Training MSE: 0.0004
+Step [4700/6957] - Training Loss: 0.0015 - Training MSE: 0.0004
+Step [4800/6957] - Training Loss: 0.0009 - Training MSE: 0.0004
+Step [4900/6957] - Training Loss: 0.0069 - Training MSE: 0.0004
+Step [5000/6957] - Training Loss: 0.0010 - Training MSE: 0.0004
+Step [5100/6957] - Training Loss: 0.0023 - Training MSE: 0.0004
+Step [5200/6957] - Training Loss: 0.0026 - Training MSE: 0.0004
+Step [5300/6957] - Training Loss: 0.0006 - Training MSE: 0.0004
+Step [5400/6957] - Training Loss: 0.0429 - Training MSE: 0.0004
+Step [5500/6957] - Training Loss: 0.0003 - Training MSE: 0.0004
+Step [5600/6957] - Training Loss: 0.0002 - Training MSE: 0.0004
+Step [5700/6957] - Training Loss: 0.0005 - Training MSE: 0.0003
+Step [5800/6957] - Training Loss: 0.0007 - Training MSE: 0.0003
+Step [5900/6957] - Training Loss: 0.0009 - Training MSE: 0.0003
+Step [6000/6957] - Training Loss: 0.0008 - Training MSE: 0.0003
+Step [6100/6957] - Training Loss: 0.0036 - Training MSE: 0.0003
+Step [6200/6957] - Training Loss: 0.0057 - Training MSE: 0.0003
+Step [6300/6957] - Training Loss: 0.0146 - Training MSE: 0.0003
+Step [6400/6957] - Training Loss: 0.0012 - Training MSE: 0.0003
+Step [6500/6957] - Training Loss: 0.0022 - Training MSE: 0.0003
+Step [6600/6957] - Training Loss: 0.0012 - Training MSE: 0.0003
+Step [6700/6957] - Training Loss: 0.0012 - Training MSE: 0.0003
+Step [6800/6957] - Training Loss: 0.0008 - Training MSE: 0.0003
+Step [6900/6957] - Training Loss: 0.0084 - Training MSE: 0.0003
+Epoch [9/20] - Training Loss: 0.0056, Training MSE: 0.0003 - Validation Loss: 0.3878, Validation MSE: 0.0242
+Step [100/6957] - Training Loss: 0.0005 - Training MSE: 0.0001
+Step [200/6957] - Training Loss: 0.0003 - Training MSE: 0.0001
+Step [300/6957] - Training Loss: 0.0007 - Training MSE: 0.0002
+Step [400/6957] - Training Loss: 0.0014 - Training MSE: 0.0002
+Step [500/6957] - Training Loss: 0.0012 - Training MSE: 0.0002
+Step [600/6957] - Training Loss: 0.0127 - Training MSE: 0.0002
+Step [700/6957] - Training Loss: 0.0010 - Training MSE: 0.0002
+Step [800/6957] - Training Loss: 0.0006 - Training MSE: 0.0002
+Step [900/6957] - Training Loss: 0.0007 - Training MSE: 0.0002
+Step [1000/6957] - Training Loss: 0.0013 - Training MSE: 0.0002
+Step [1100/6957] - Training Loss: 0.0009 - Training MSE: 0.0002
+Step [1200/6957] - Training Loss: 0.0011 - Training MSE: 0.0002
+Step [1300/6957] - Training Loss: 0.0020 - Training MSE: 0.0002
+Step [1400/6957] - Training Loss: 0.0016 - Training MSE: 0.0002
+Step [1500/6957] - Training Loss: 0.0007 - Training MSE: 0.0002
+Step [1600/6957] - Training Loss: 0.0011 - Training MSE: 0.0002
+Step [1700/6957] - Training Loss: 0.0019 - Training MSE: 0.0002
+Step [1800/6957] - Training Loss: 0.0010 - Training MSE: 0.0002
+Step [1900/6957] - Training Loss: 0.0015 - Training MSE: 0.0002
+Step [2000/6957] - Training Loss: 0.0002 - Training MSE: 0.0002
+Step [2100/6957] - Training Loss: 0.0006 - Training MSE: 0.0002
+Step [2200/6957] - Training Loss: 0.0373 - Training MSE: 0.0002
+Step [2300/6957] - Training Loss: 0.0008 - Training MSE: 0.0002
+Step [2400/6957] - Training Loss: 0.0281 - Training MSE: 0.0003
+Step [2500/6957] - Training Loss: 0.0002 - Training MSE: 0.0003
+Step [2600/6957] - Training Loss: 0.0036 - Training MSE: 0.0003
+Step [2700/6957] - Training Loss: 0.0058 - Training MSE: 0.0003
+Step [2800/6957] - Training Loss: 0.0010 - Training MSE: 0.0003
+Step [2900/6957] - Training Loss: 0.0014 - Training MSE: 0.0003
+Step [3000/6957] - Training Loss: 0.0018 - Training MSE: 0.0003
+Step [3100/6957] - Training Loss: 0.0008 - Training MSE: 0.0003
+Step [3200/6957] - Training Loss: 0.0027 - Training MSE: 0.0003
+Step [3300/6957] - Training Loss: 0.0042 - Training MSE: 0.0003
+Step [3400/6957] - Training Loss: 0.0020 - Training MSE: 0.0003
+Step [3500/6957] - Training Loss: 0.0004 - Training MSE: 0.0003
+Step [3600/6957] - Training Loss: 0.0005 - Training MSE: 0.0003
+Step [3700/6957] - Training Loss: 0.0004 - Training MSE: 0.0003
+Step [3800/6957] - Training Loss: 0.0006 - Training MSE: 0.0003
+Step [3900/6957] - Training Loss: 0.0004 - Training MSE: 0.0003
+Step [4000/6957] - Training Loss: 0.0003 - Training MSE: 0.0003
+Step [4100/6957] - Training Loss: 0.0005 - Training MSE: 0.0003
+Step [4200/6957] - Training Loss: 0.0004 - Training MSE: 0.0003
+Step [4300/6957] - Training Loss: 0.0001 - Training MSE: 0.0003
+Step [4400/6957] - Training Loss: 0.0001 - Training MSE: 0.0003
+Step [4500/6957] - Training Loss: 0.0002 - Training MSE: 0.0003
+Step [4600/6957] - Training Loss: 0.0011 - Training MSE: 0.0003
+Step [4700/6957] - Training Loss: 0.0007 - Training MSE: 0.0003
+Step [4800/6957] - Training Loss: 0.0002 - Training MSE: 0.0003
+Step [4900/6957] - Training Loss: 0.0073 - Training MSE: 0.0003
+Step [5000/6957] - Training Loss: 0.0017 - Training MSE: 0.0003
+Step [5100/6957] - Training Loss: 0.0002 - Training MSE: 0.0003
+Step [5200/6957] - Training Loss: 0.0035 - Training MSE: 0.0003
+Step [5300/6957] - Training Loss: 0.0003 - Training MSE: 0.0003
+Step [5400/6957] - Training Loss: 0.0013 - Training MSE: 0.0003
+Step [5500/6957] - Training Loss: 0.0004 - Training MSE: 0.0003
+Step [5600/6957] - Training Loss: 0.0131 - Training MSE: 0.0003
+Step [5700/6957] - Training Loss: 0.0004 - Training MSE: 0.0003
+Step [5800/6957] - Training Loss: 0.0015 - Training MSE: 0.0003
+Step [5900/6957] - Training Loss: 0.0001 - Training MSE: 0.0003
+Step [6000/6957] - Training Loss: 0.0008 - Training MSE: 0.0003
+Step [6100/6957] - Training Loss: 0.0243 - Training MSE: 0.0003
+Step [6200/6957] - Training Loss: 0.0030 - Training MSE: 0.0003
+Step [6300/6957] - Training Loss: 0.0004 - Training MSE: 0.0003
+Step [6400/6957] - Training Loss: 0.0007 - Training MSE: 0.0003
+Step [6500/6957] - Training Loss: 0.0003 - Training MSE: 0.0003
+Step [6600/6957] - Training Loss: 0.0004 - Training MSE: 0.0003
+Step [6700/6957] - Training Loss: 0.0006 - Training MSE: 0.0003
+Step [6800/6957] - Training Loss: 0.0014 - Training MSE: 0.0003
+Step [6900/6957] - Training Loss: 0.0005 - Training MSE: 0.0003
+Epoch [10/20] - Training Loss: 0.0043, Training MSE: 0.0003 - Validation Loss: 0.3933, Validation MSE: 0.0246
+Step [100/6957] - Training Loss: 0.0002 - Training MSE: 0.0002
+Step [200/6957] - Training Loss: 0.0020 - Training MSE: 0.0002
+Step [300/6957] - Training Loss: 0.0011 - Training MSE: 0.0002
+Step [400/6957] - Training Loss: 0.0005 - Training MSE: 0.0002
+Step [500/6957] - Training Loss: 0.0008 - Training MSE: 0.0002
+Step [600/6957] - Training Loss: 0.0004 - Training MSE: 0.0002
+Step [700/6957] - Training Loss: 0.0025 - Training MSE: 0.0002
+Step [800/6957] - Training Loss: 0.0004 - Training MSE: 0.0002
+Step [900/6957] - Training Loss: 0.0002 - Training MSE: 0.0002
+Step [1000/6957] - Training Loss: 0.0001 - Training MSE: 0.0001
+Step [1100/6957] - Training Loss: 0.0002 - Training MSE: 0.0001
+Step [1200/6957] - Training Loss: 0.0002 - Training MSE: 0.0001
+Step [1300/6957] - Training Loss: 0.0028 - Training MSE: 0.0002
+Step [1400/6957] - Training Loss: 0.0007 - Training MSE: 0.0002
+Step [1500/6957] - Training Loss: 0.0157 - Training MSE: 0.0002
+Step [1600/6957] - Training Loss: 0.0002 - Training MSE: 0.0002
+Step [1700/6957] - Training Loss: 0.0012 - Training MSE: 0.0002
+Step [1800/6957] - Training Loss: 0.0003 - Training MSE: 0.0002
+Step [1900/6957] - Training Loss: 0.0001 - Training MSE: 0.0002
+Step [2000/6957] - Training Loss: 0.0002 - Training MSE: 0.0002
+Step [2100/6957] - Training Loss: 0.0002 - Training MSE: 0.0002
+Step [2200/6957] - Training Loss: 0.0002 - Training MSE: 0.0002
+Step [2300/6957] - Training Loss: 0.0016 - Training MSE: 0.0002
+Step [2400/6957] - Training Loss: 0.0276 - Training MSE: 0.0002
+Step [2500/6957] - Training Loss: 0.0016 - Training MSE: 0.0002
+Step [2600/6957] - Training Loss: 0.0012 - Training MSE: 0.0002
+Step [2700/6957] - Training Loss: 0.0003 - Training MSE: 0.0002
+Step [2800/6957] - Training Loss: 0.0002 - Training MSE: 0.0002
+Step [2900/6957] - Training Loss: 0.0003 - Training MSE: 0.0002
+Step [3000/6957] - Training Loss: 0.0017 - Training MSE: 0.0002
+Step [3100/6957] - Training Loss: 0.0003 - Training MSE: 0.0002
+Step [3200/6957] - Training Loss: 0.0003 - Training MSE: 0.0002
+Step [3300/6957] - Training Loss: 0.0006 - Training MSE: 0.0002
+Step [3400/6957] - Training Loss: 0.0006 - Training MSE: 0.0002
+Step [3500/6957] - Training Loss: 0.0002 - Training MSE: 0.0002
+Step [3600/6957] - Training Loss: 0.0006 - Training MSE: 0.0002
+Step [3700/6957] - Training Loss: 0.0075 - Training MSE: 0.0002
+Step [3800/6957] - Training Loss: 0.0003 - Training MSE: 0.0002
+Step [3900/6957] - Training Loss: 0.0008 - Training MSE: 0.0002
+Step [4000/6957] - Training Loss: 0.0006 - Training MSE: 0.0002
+Step [4100/6957] - Training Loss: 0.0001 - Training MSE: 0.0002
+Step [4200/6957] - Training Loss: 0.0024 - Training MSE: 0.0002
+Step [4300/6957] - Training Loss: 0.0006 - Training MSE: 0.0002
+Step [4400/6957] - Training Loss: 0.0020 - Training MSE: 0.0002
+Step [4500/6957] - Training Loss: 0.0002 - Training MSE: 0.0002
+Step [4600/6957] - Training Loss: 0.0001 - Training MSE: 0.0002
+Step [4700/6957] - Training Loss: 0.0020 - Training MSE: 0.0002
+Step [4800/6957] - Training Loss: 0.0002 - Training MSE: 0.0002
+Step [4900/6957] - Training Loss: 0.0012 - Training MSE: 0.0002
+Step [5000/6957] - Training Loss: 0.0017 - Training MSE: 0.0002
+Step [5100/6957] - Training Loss: 0.0010 - Training MSE: 0.0002
+Step [5200/6957] - Training Loss: 0.0004 - Training MSE: 0.0002
+Step [5300/6957] - Training Loss: 0.0006 - Training MSE: 0.0002
+Step [5400/6957] - Training Loss: 0.0003 - Training MSE: 0.0002
+Step [5500/6957] - Training Loss: 0.0005 - Training MSE: 0.0002
+Step [5600/6957] - Training Loss: 0.0012 - Training MSE: 0.0002
+Step [5700/6957] - Training Loss: 0.0008 - Training MSE: 0.0002
+Step [5800/6957] - Training Loss: 0.0003 - Training MSE: 0.0002
+Step [5900/6957] - Training Loss: 0.0001 - Training MSE: 0.0002
+Step [6000/6957] - Training Loss: 0.0007 - Training MSE: 0.0002
+Step [6100/6957] - Training Loss: 0.0003 - Training MSE: 0.0002
+Step [6200/6957] - Training Loss: 0.0036 - Training MSE: 0.0002
+Step [6300/6957] - Training Loss: 0.0002 - Training MSE: 0.0002
+Step [6400/6957] - Training Loss: 0.0013 - Training MSE: 0.0002
+Step [6500/6957] - Training Loss: 0.0220 - Training MSE: 0.0002
+Step [6600/6957] - Training Loss: 0.0001 - Training MSE: 0.0002
+Step [6700/6957] - Training Loss: 0.0301 - Training MSE: 0.0002
+Step [6800/6957] - Training Loss: 0.0064 - Training MSE: 0.0002
+Step [6900/6957] - Training Loss: 0.0029 - Training MSE: 0.0002
+Epoch [11/20] - Training Loss: 0.0028, Training MSE: 0.0002 - Validation Loss: 0.3914, Validation MSE: 0.0245
+Step [100/6957] - Training Loss: 0.0132 - Training MSE: 0.0003
+Step [200/6957] - Training Loss: 0.0018 - Training MSE: 0.0002
+Step [300/6957] - Training Loss: 0.0014 - Training MSE: 0.0003
+Step [400/6957] - Training Loss: 0.0011 - Training MSE: 0.0003
+Step [500/6957] - Training Loss: 0.0011 - Training MSE: 0.0002
+Step [600/6957] - Training Loss: 0.0002 - Training MSE: 0.0002
+Step [700/6957] - Training Loss: 0.0005 - Training MSE: 0.0002
+Step [800/6957] - Training Loss: 0.0014 - Training MSE: 0.0002
+Step [900/6957] - Training Loss: 0.0004 - Training MSE: 0.0002
+Step [1000/6957] - Training Loss: 0.0001 - Training MSE: 0.0002
+Step [1100/6957] - Training Loss: 0.0000 - Training MSE: 0.0002
+Step [1200/6957] - Training Loss: 0.0001 - Training MSE: 0.0001
+Step [1300/6957] - Training Loss: 0.0003 - Training MSE: 0.0001
+Step [1400/6957] - Training Loss: 0.0009 - Training MSE: 0.0001
+Step [1500/6957] - Training Loss: 0.0009 - Training MSE: 0.0001
+Step [1600/6957] - Training Loss: 0.0002 - Training MSE: 0.0001
+Step [1700/6957] - Training Loss: 0.0001 - Training MSE: 0.0001
+Step [1800/6957] - Training Loss: 0.0002 - Training MSE: 0.0001
+Step [1900/6957] - Training Loss: 0.0005 - Training MSE: 0.0001
+Step [2000/6957] - Training Loss: 0.0011 - Training MSE: 0.0001
+Step [2100/6957] - Training Loss: 0.0401 - Training MSE: 0.0002
+Step [2200/6957] - Training Loss: 0.0005 - Training MSE: 0.0002
+Step [2300/6957] - Training Loss: 0.0164 - Training MSE: 0.0002
+Step [2400/6957] - Training Loss: 0.0002 - Training MSE: 0.0002
+Step [2500/6957] - Training Loss: 0.0001 - Training MSE: 0.0001
+Step [2600/6957] - Training Loss: 0.0002 - Training MSE: 0.0001
+Step [2700/6957] - Training Loss: 0.0002 - Training MSE: 0.0001
+Step [2800/6957] - Training Loss: 0.0003 - Training MSE: 0.0001
+Step [2900/6957] - Training Loss: 0.0002 - Training MSE: 0.0001
+Step [3000/6957] - Training Loss: 0.0001 - Training MSE: 0.0001
+Step [3100/6957] - Training Loss: 0.0002 - Training MSE: 0.0001
+Step [3200/6957] - Training Loss: 0.0002 - Training MSE: 0.0001
+Step [3300/6957] - Training Loss: 0.0001 - Training MSE: 0.0001
+Step [3400/6957] - Training Loss: 0.0004 - Training MSE: 0.0001
+Step [3500/6957] - Training Loss: 0.0003 - Training MSE: 0.0001
+Step [3600/6957] - Training Loss: 0.0004 - Training MSE: 0.0001
+Step [3700/6957] - Training Loss: 0.0006 - Training MSE: 0.0001
+Step [3800/6957] - Training Loss: 0.0008 - Training MSE: 0.0001
+Step [3900/6957] - Training Loss: 0.0011 - Training MSE: 0.0001
+Step [4000/6957] - Training Loss: 0.0001 - Training MSE: 0.0001
+Step [4100/6957] - Training Loss: 0.0003 - Training MSE: 0.0001
+Step [4200/6957] - Training Loss: 0.0001 - Training MSE: 0.0001
+Step [4300/6957] - Training Loss: 0.0009 - Training MSE: 0.0001
+Step [4400/6957] - Training Loss: 0.0001 - Training MSE: 0.0001
+Step [4500/6957] - Training Loss: 0.0003 - Training MSE: 0.0001
+Step [4600/6957] - Training Loss: 0.0002 - Training MSE: 0.0001
+Step [4700/6957] - Training Loss: 0.0008 - Training MSE: 0.0001
+Step [4800/6957] - Training Loss: 0.0005 - Training MSE: 0.0001
+Step [4900/6957] - Training Loss: 0.0001 - Training MSE: 0.0001
+Step [5000/6957] - Training Loss: 0.0003 - Training MSE: 0.0001
+Step [5100/6957] - Training Loss: 0.0001 - Training MSE: 0.0001
+Step [5200/6957] - Training Loss: 0.0001 - Training MSE: 0.0001
+Step [5300/6957] - Training Loss: 0.0006 - Training MSE: 0.0001
+Step [5400/6957] - Training Loss: 0.0001 - Training MSE: 0.0001
+Step [5500/6957] - Training Loss: 0.0003 - Training MSE: 0.0001
+Step [5600/6957] - Training Loss: 0.0002 - Training MSE: 0.0001
+Step [5700/6957] - Training Loss: 0.0001 - Training MSE: 0.0001
+Step [5800/6957] - Training Loss: 0.0002 - Training MSE: 0.0001
+Step [5900/6957] - Training Loss: 0.0001 - Training MSE: 0.0001
+Step [6000/6957] - Training Loss: 0.0014 - Training MSE: 0.0001
+Step [6100/6957] - Training Loss: 0.0003 - Training MSE: 0.0001
+Step [6200/6957] - Training Loss: 0.0001 - Training MSE: 0.0001
+Step [6300/6957] - Training Loss: 0.0001 - Training MSE: 0.0001
+Step [6400/6957] - Training Loss: 0.0003 - Training MSE: 0.0001
+Step [6500/6957] - Training Loss: 0.0009 - Training MSE: 0.0001
+Step [6600/6957] - Training Loss: 0.0006 - Training MSE: 0.0001
+Step [6700/6957] - Training Loss: 0.0003 - Training MSE: 0.0001
+Step [6800/6957] - Training Loss: 0.0002 - Training MSE: 0.0001
+Step [6900/6957] - Training Loss: 0.0004 - Training MSE: 0.0001
+Epoch [12/20] - Training Loss: 0.0017, Training MSE: 0.0001 - Validation Loss: 0.4000, Validation MSE: 0.0250
+Step [100/6957] - Training Loss: 0.0005 - Training MSE: 0.0001
+Step [200/6957] - Training Loss: 0.0002 - Training MSE: 0.0001
+Step [300/6957] - Training Loss: 0.0001 - Training MSE: 0.0001
+Step [400/6957] - Training Loss: 0.0001 - Training MSE: 0.0001
+Step [500/6957] - Training Loss: 0.0001 - Training MSE: 0.0001
+Step [600/6957] - Training Loss: 0.0001 - Training MSE: 0.0001
+Step [700/6957] - Training Loss: 0.0001 - Training MSE: 0.0001
+Step [800/6957] - Training Loss: 0.0002 - Training MSE: 0.0001
+Step [900/6957] - Training Loss: 0.0008 - Training MSE: 0.0001
+Step [1000/6957] - Training Loss: 0.0001 - Training MSE: 0.0001
+Step [1100/6957] - Training Loss: 0.0001 - Training MSE: 0.0001
+Step [1200/6957] - Training Loss: 0.0004 - Training MSE: 0.0001
+Step [1300/6957] - Training Loss: 0.0001 - Training MSE: 0.0001
+Step [1400/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1500/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1600/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [1700/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [1800/6957] - Training Loss: 0.0002 - Training MSE: 0.0000
+Step [1900/6957] - Training Loss: 0.0020 - Training MSE: 0.0001
+Step [2000/6957] - Training Loss: 0.0003 - Training MSE: 0.0001
+Step [2100/6957] - Training Loss: 0.0005 - Training MSE: 0.0001
+Step [2200/6957] - Training Loss: 0.0002 - Training MSE: 0.0001
+Step [2300/6957] - Training Loss: 0.0605 - Training MSE: 0.0001
+Step [2400/6957] - Training Loss: 0.0001 - Training MSE: 0.0001
+Step [2500/6957] - Training Loss: 0.0010 - Training MSE: 0.0001
+Step [2600/6957] - Training Loss: 0.0002 - Training MSE: 0.0001
+Step [2700/6957] - Training Loss: 0.0002 - Training MSE: 0.0001
+Step [2800/6957] - Training Loss: 0.0008 - Training MSE: 0.0001
+Step [2900/6957] - Training Loss: 0.0002 - Training MSE: 0.0001
+Step [3000/6957] - Training Loss: 0.0001 - Training MSE: 0.0001
+Step [3100/6957] - Training Loss: 0.0001 - Training MSE: 0.0001
+Step [3200/6957] - Training Loss: 0.0147 - Training MSE: 0.0001
+Step [3300/6957] - Training Loss: 0.0001 - Training MSE: 0.0001
+Step [3400/6957] - Training Loss: 0.0001 - Training MSE: 0.0001
+Step [3500/6957] - Training Loss: 0.0039 - Training MSE: 0.0001
+Step [3600/6957] - Training Loss: 0.0001 - Training MSE: 0.0001
+Step [3700/6957] - Training Loss: 0.0004 - Training MSE: 0.0001
+Step [3800/6957] - Training Loss: 0.0002 - Training MSE: 0.0001
+Step [3900/6957] - Training Loss: 0.0002 - Training MSE: 0.0001
+Step [4000/6957] - Training Loss: 0.0003 - Training MSE: 0.0001
+Step [4100/6957] - Training Loss: 0.0002 - Training MSE: 0.0001
+Step [4200/6957] - Training Loss: 0.0002 - Training MSE: 0.0001
+Step [4300/6957] - Training Loss: 0.0001 - Training MSE: 0.0001
+Step [4400/6957] - Training Loss: 0.0001 - Training MSE: 0.0001
+Step [4500/6957] - Training Loss: 0.0002 - Training MSE: 0.0001
+Step [4600/6957] - Training Loss: 0.0001 - Training MSE: 0.0001
+Step [4700/6957] - Training Loss: 0.0001 - Training MSE: 0.0001
+Step [4800/6957] - Training Loss: 0.0000 - Training MSE: 0.0001
+Step [4900/6957] - Training Loss: 0.0007 - Training MSE: 0.0001
+Step [5000/6957] - Training Loss: 0.0018 - Training MSE: 0.0001
+Step [5100/6957] - Training Loss: 0.0002 - Training MSE: 0.0001
+Step [5200/6957] - Training Loss: 0.0002 - Training MSE: 0.0001
+Step [5300/6957] - Training Loss: 0.0002 - Training MSE: 0.0001
+Step [5400/6957] - Training Loss: 0.0005 - Training MSE: 0.0001
+Step [5500/6957] - Training Loss: 0.0006 - Training MSE: 0.0001
+Step [5600/6957] - Training Loss: 0.0002 - Training MSE: 0.0001
+Step [5700/6957] - Training Loss: 0.0004 - Training MSE: 0.0001
+Step [5800/6957] - Training Loss: 0.0002 - Training MSE: 0.0001
+Step [5900/6957] - Training Loss: 0.0018 - Training MSE: 0.0001
+Step [6000/6957] - Training Loss: 0.0051 - Training MSE: 0.0001
+Step [6100/6957] - Training Loss: 0.0001 - Training MSE: 0.0001
+Step [6200/6957] - Training Loss: 0.0001 - Training MSE: 0.0001
+Step [6300/6957] - Training Loss: 0.0001 - Training MSE: 0.0001
+Step [6400/6957] - Training Loss: 0.0000 - Training MSE: 0.0001
+Step [6500/6957] - Training Loss: 0.0000 - Training MSE: 0.0001
+Step [6600/6957] - Training Loss: 0.0001 - Training MSE: 0.0001
+Step [6700/6957] - Training Loss: 0.0001 - Training MSE: 0.0001
+Step [6800/6957] - Training Loss: 0.0001 - Training MSE: 0.0001
+Step [6900/6957] - Training Loss: 0.0000 - Training MSE: 0.0001
+Epoch [13/20] - Training Loss: 0.0011, Training MSE: 0.0001 - Validation Loss: 0.3817, Validation MSE: 0.0239
+Step [100/6957] - Training Loss: 0.0003 - Training MSE: 0.0001
+Step [200/6957] - Training Loss: 0.0001 - Training MSE: 0.0001
+Step [300/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [400/6957] - Training Loss: 0.0009 - Training MSE: 0.0000
+Step [500/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [600/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [700/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [800/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [900/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1000/6957] - Training Loss: 0.0003 - Training MSE: 0.0000
+Step [1100/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [1200/6957] - Training Loss: 0.0002 - Training MSE: 0.0000
+Step [1300/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1400/6957] - Training Loss: 0.0004 - Training MSE: 0.0000
+Step [1500/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [1600/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1700/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [1800/6957] - Training Loss: 0.0004 - Training MSE: 0.0000
+Step [1900/6957] - Training Loss: 0.0005 - Training MSE: 0.0000
+Step [2000/6957] - Training Loss: 0.0010 - Training MSE: 0.0000
+Step [2100/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2200/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [2300/6957] - Training Loss: 0.0002 - Training MSE: 0.0000
+Step [2400/6957] - Training Loss: 0.0011 - Training MSE: 0.0000
+Step [2500/6957] - Training Loss: 0.0002 - Training MSE: 0.0000
+Step [2600/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [2700/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2800/6957] - Training Loss: 0.0002 - Training MSE: 0.0000
+Step [2900/6957] - Training Loss: 0.0002 - Training MSE: 0.0000
+Step [3000/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [3100/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [3200/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [3300/6957] - Training Loss: 0.0003 - Training MSE: 0.0000
+Step [3400/6957] - Training Loss: 0.0174 - Training MSE: 0.0000
+Step [3500/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [3600/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [3700/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [3800/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [3900/6957] - Training Loss: 0.0003 - Training MSE: 0.0000
+Step [4000/6957] - Training Loss: 0.0002 - Training MSE: 0.0000
+Step [4100/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [4200/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [4300/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [4400/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4500/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4600/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4700/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4800/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4900/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5000/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [5100/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [5200/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5300/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [5400/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [5500/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [5600/6957] - Training Loss: 0.0005 - Training MSE: 0.0000
+Step [5700/6957] - Training Loss: 0.0458 - Training MSE: 0.0000
+Step [5800/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [5900/6957] - Training Loss: 0.0002 - Training MSE: 0.0000
+Step [6000/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6100/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [6200/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6300/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [6400/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6500/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6600/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [6700/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [6800/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6900/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Epoch [14/20] - Training Loss: 0.0006, Training MSE: 0.0000 - Validation Loss: 0.3896, Validation MSE: 0.0243
+Step [100/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [200/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [300/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [400/6957] - Training Loss: 0.0003 - Training MSE: 0.0000
+Step [500/6957] - Training Loss: 0.0067 - Training MSE: 0.0000
+Step [600/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [700/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [800/6957] - Training Loss: 0.0004 - Training MSE: 0.0000
+Step [900/6957] - Training Loss: 0.0005 - Training MSE: 0.0000
+Step [1000/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [1100/6957] - Training Loss: 0.0002 - Training MSE: 0.0000
+Step [1200/6957] - Training Loss: 0.0002 - Training MSE: 0.0000
+Step [1300/6957] - Training Loss: 0.0002 - Training MSE: 0.0000
+Step [1400/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1500/6957] - Training Loss: 0.0002 - Training MSE: 0.0000
+Step [1600/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [1700/6957] - Training Loss: 0.0002 - Training MSE: 0.0000
+Step [1800/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1900/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [2000/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [2100/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [2200/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2300/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2400/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [2500/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [2600/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2700/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [2800/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [2900/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3000/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [3100/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3200/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3300/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3400/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3500/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3600/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3700/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3800/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3900/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4000/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [4100/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4200/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4300/6957] - Training Loss: 0.0002 - Training MSE: 0.0000
+Step [4400/6957] - Training Loss: 0.0003 - Training MSE: 0.0000
+Step [4500/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [4600/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4700/6957] - Training Loss: 0.0002 - Training MSE: 0.0000
+Step [4800/6957] - Training Loss: 0.0002 - Training MSE: 0.0000
+Step [4900/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [5000/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [5100/6957] - Training Loss: 0.0002 - Training MSE: 0.0000
+Step [5200/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5300/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5400/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5500/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [5600/6957] - Training Loss: 0.0037 - Training MSE: 0.0000
+Step [5700/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [5800/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5900/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6000/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [6100/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6200/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6300/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6400/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6500/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6600/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6700/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6800/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6900/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Epoch [15/20] - Training Loss: 0.0003, Training MSE: 0.0000 - Validation Loss: 0.3805, Validation MSE: 0.0238
+Step [100/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [200/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [300/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [400/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [500/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [600/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [700/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [800/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [900/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1000/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1100/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [1200/6957] - Training Loss: 0.0002 - Training MSE: 0.0000
+Step [1300/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [1400/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1500/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1600/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1700/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1800/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1900/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2000/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2100/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2200/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [2300/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [2400/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2500/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2600/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [2700/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [2800/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [2900/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3000/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3100/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3200/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3300/6957] - Training Loss: 0.0002 - Training MSE: 0.0000
+Step [3400/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [3500/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3600/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [3700/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3800/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3900/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4000/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [4100/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4200/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4300/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4400/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4500/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4600/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4700/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4800/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4900/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5000/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5100/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5200/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5300/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5400/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [5500/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5600/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5700/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5800/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5900/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [6000/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [6100/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6200/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6300/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6400/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6500/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6600/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6700/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6800/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6900/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Epoch [16/20] - Training Loss: 0.0002, Training MSE: 0.0000 - Validation Loss: 0.3771, Validation MSE: 0.0236
+Step [100/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [200/6957] - Training Loss: 0.0004 - Training MSE: 0.0000
+Step [300/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [400/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [500/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [600/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [700/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [800/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [900/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1000/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1100/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1200/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1300/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1400/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1500/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1600/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1700/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1800/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1900/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2000/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2100/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2200/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2300/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2400/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2500/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2600/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2700/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2800/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2900/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3000/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3100/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3200/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3300/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3400/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3500/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3600/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3700/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3800/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3900/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4000/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [4100/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4200/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4300/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4400/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4500/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4600/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4700/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4800/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4900/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5000/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5100/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5200/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5300/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5400/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5500/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5600/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5700/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5800/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5900/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6000/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6100/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6200/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6300/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6400/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6500/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6600/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6700/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6800/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [6900/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Epoch [17/20] - Training Loss: 0.0001, Training MSE: 0.0000 - Validation Loss: 0.3723, Validation MSE: 0.0233
+Step [100/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [200/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [300/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [400/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [500/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [600/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [700/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [800/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [900/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1000/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1100/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1200/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1300/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1400/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1500/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1600/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1700/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1800/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1900/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2000/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2100/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2200/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2300/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2400/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2500/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2600/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2700/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2800/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2900/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3000/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3100/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3200/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3300/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3400/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3500/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3600/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3700/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3800/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3900/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4000/6957] - Training Loss: 0.0001 - Training MSE: 0.0000
+Step [4100/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4200/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4300/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4400/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4500/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4600/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4700/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4800/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4900/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5000/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5100/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5200/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5300/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5400/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5500/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5600/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5700/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5800/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5900/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6000/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6100/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6200/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6300/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6400/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6500/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6600/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6700/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6800/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6900/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Epoch [18/20] - Training Loss: 0.0000, Training MSE: 0.0000 - Validation Loss: 0.3721, Validation MSE: 0.0233
+Step [100/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [200/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [300/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [400/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [500/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [600/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [700/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [800/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [900/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1000/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1100/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1200/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1300/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1400/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1500/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1600/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1700/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1800/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1900/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2000/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2100/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2200/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2300/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2400/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2500/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2600/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2700/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2800/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2900/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3000/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3100/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3200/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3300/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3400/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3500/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3600/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3700/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3800/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3900/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4000/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4100/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4200/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4300/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4400/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4500/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4600/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4700/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4800/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4900/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5000/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5100/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5200/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5300/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5400/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5500/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5600/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5700/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5800/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5900/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6000/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6100/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6200/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6300/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6400/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6500/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6600/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6700/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6800/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6900/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Epoch [19/20] - Training Loss: 0.0000, Training MSE: 0.0000 - Validation Loss: 0.3732, Validation MSE: 0.0233
+Step [100/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [200/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [300/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [400/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [500/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [600/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [700/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [800/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [900/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1000/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1100/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1200/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1300/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1400/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1500/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1600/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1700/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1800/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [1900/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2000/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2100/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2200/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2300/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2400/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2500/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2600/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2700/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2800/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [2900/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3000/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3100/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3200/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3300/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3400/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3500/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3600/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3700/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3800/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [3900/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4000/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4100/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4200/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4300/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4400/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4500/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4600/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4700/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4800/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [4900/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5000/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5100/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5200/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5300/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5400/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5500/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5600/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5700/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5800/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [5900/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6000/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6100/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6200/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6300/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6400/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6500/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6600/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6700/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6800/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Step [6900/6957] - Training Loss: 0.0000 - Training MSE: 0.0000
+Epoch [20/20] - Training Loss: 0.0000, Training MSE: 0.0000 - Validation Loss: 0.3730, Validation MSE: 0.0233
+wandb: 🚀 View run HCPflat_large_gsrFalse__beta_age_HCPFT at: https://stability.wandb.io/ckadirt/fMRI-foundation-model/runs/HCPflat_large_gsrFalse__beta_age_HCPFT_d344d64d-8300-4465-8e75-b35200117944
+wandb: Find logs at: wandb/run-20241127_023614-HCPflat_large_gsrFalse__beta_age_HCPFT_d344d64d-8300-4465-8e75-b35200117944/logs

541360.err

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:992c93a72705f7b1913906a6154cace15e243ed86564339518ce4fe1f3313168
+size 13421980